Upload folder using huggingface_hub

modules/dropbox/discources.py

@@ -1,82 +1,94 @@
-import os
+import asyncio
 import json
-
-def fetch_all_discourses(folder_path="data/dropbox/discourses"):
+import logging
+from datetime import datetime, timedelta
+from typing import List, Optional
+import dropbox
+from modules.dropbox.client import dbx
+
+# Logger
+logger = logging.getLogger(__name__)
+logger.setLevel(logging.INFO)
+
+# Cache: key = folder_path, value = {"timestamp": datetime, "data": List[dict]}
+_discourse_cache: dict[str, dict] = {}
+CACHE_TTL = timedelta(hours=1)
+FOLDER_PATH = "/_discourses"
+
+async def fetch_discourses_from_dropbox() -> List[dict]:
     """
-    Loads all JSON discourse files from the specified folder and returns
-    them as a list of Python dictionaries.
-
-    Each file should contain a valid JSON object representing one topic.
+    Fetch all discourse JSONs for a scripture from Dropbox with caching.
+    Expects files in "/_discourses/".
     """
-    discourses = []
-
-    if not os.path.exists(folder_path):
-        print(f"⚠️ Folder not found: {folder_path}")
+    loop = asyncio.get_running_loop()
+    folder_path = FOLDER_PATH
+
+    # Check cache
+    cache_entry = _discourse_cache.get(folder_path)
+    if cache_entry:
+        age = datetime.now() - cache_entry["timestamp"]
+        if age < CACHE_TTL:
+            logger.info(f"Using cached discourses for '{folder_path}' (age={age})")
+            return cache_entry["data"]
+
+    logger.info(f"Fetching discourses from Dropbox folder '{folder_path}'")
+    discourses: List[dict] = []
+
+    try:
+        # List folder contents (synchronously in executor)
+        res = await loop.run_in_executor(None, dbx.files_list_folder, folder_path)
+        for entry in res.entries:
+            if isinstance(entry, dropbox.files.FileMetadata) and entry.name.lower().endswith(".json"):
+                metadata, fres = await loop.run_in_executor(
+                    None, dbx.files_download, f"{folder_path}/{entry.name}"
+                )
+                data = fres.content.decode("utf-8")
+                discourses.append(json.loads(data))
+
+        # Update cache
+        _discourse_cache[folder_path] = {"timestamp": datetime.now(), "data": discourses}
+        logger.info(f"Cached {len(discourses)} discourses for '{folder_path}'")
         return discourses
 
-    for filename in os.listdir(folder_path):
-        if filename.lower().endswith(".json"):
-            file_path = os.path.join(folder_path, filename)
-            try:
-                with open(file_path, "r", encoding="utf-8") as f:
-                    data = json.load(f)
-                    discourses.append(data)
-            except Exception as e:
-                print(f"❌ Error loading {filename}: {e}")
+    except Exception as e:
+        logger.error(f"Error fetching discourses from '{folder_path}'", exc_info=e)
+        # fallback to cached data if available
+        if cache_entry:
+            logger.warning(f"Returning stale cached discourses for '{folder_path}'")
+            return cache_entry["data"]
+        else:
+            logger.warning(f"No cached discourses available for '{folder_path}'")
+            return []
 
-    return discourses
 
-def get_discourse_summaries(folder_path="data/dropbox/discourses", page=1, per_page=10):
+async def get_discourse_summaries(page: int = 1, per_page: int = 10):
     """
-    Returns a paginated list of discourse topic summaries.
-    
-    Each summary contains:
-      - id
-      - topic_name
-      - thumbnail_url
-    
-    The results are sorted alphabetically by topic_name.
-
-    Args:
-        folder_path (str): Path to the folder containing JSON files.
-        page (int): Page number (1-indexed).
-        per_page (int): Number of items per page.
+    Returns paginated summaries: id, topic_name, thumbnail_url.
+    Sorted by topic_name.
     """
-    summaries = []
-
-    if not os.path.exists(folder_path):
-        print(f"⚠️ Folder not found: {folder_path}")
-        return []
-
-    # Load and extract only minimal info
-    for filename in os.listdir(folder_path):
-        if filename.lower().endswith(".json"):
-            file_path = os.path.join(folder_path, filename)
-            try:
-                with open(file_path, "r", encoding="utf-8") as f:
-                    data = json.load(f)
-                    summaries.append({
-                        "id": data.get("id"),
-                        "topic_name": data.get("topic_name"),
-                        "thumbnail_url": data.get("thumbnail_url")
-                    })
-            except Exception as e:
-                print(f"❌ Error loading {filename}: {e}")
-
-    # Sort by topic name
+    all_discourses = await fetch_discourses_from_dropbox()
+
+    # Build summaries
+    summaries = [
+        {
+            "id": d.get("id"),
+            "topic_name": d.get("topic_name"),
+            "thumbnail_url": d.get("thumbnail_url"),
+        }
+        for d in all_discourses
+    ]
+
     summaries.sort(key=lambda x: (x.get("topic_name") or "").lower())
 
-    # Pagination logic
+    # Pagination
     total_items = len(summaries)
-    total_pages = (total_items + per_page - 1) // per_page  # ceil division
-
+    total_pages = (total_items + per_page - 1) // per_page
     if page < 1 or page > total_pages:
-        print(f"⚠️ Invalid page number {page}. Must be between 1 and {total_pages}.")
-        return []
+        logger.warning(f"Invalid page {page}. Must be between 1 and {total_pages}")
+        return {"page": page, "per_page": per_page, "total_pages": total_pages, "total_items": total_items, "data": []}
 
     start = (page - 1) * per_page
     end = start + per_page
-
     paginated = summaries[start:end]
 
     return {
@@ -84,31 +96,43 @@ def get_discourse_summaries(folder_path="data/dropbox/discourses", page=1, per_p
         "per_page": per_page,
         "total_pages": total_pages,
         "total_items": total_items,
-        "data": paginated
+        "data": paginated,
     }
 
 
-import os
-import json
-
-def get_discourse_by_id(topic_id: int, folder_path="data/dropbox/discourses"):
+async def get_discourse_by_id(topic_id: int) -> Optional[dict]:
     """
-    Returns the full discourse JSON for a given topic_id.
-    Returns None if not found.
+    Fetch a single discourse JSON by topic_id from Dropbox.
+    Uses in-memory caching per file.
     """
-    if not os.path.exists(folder_path):
-        print(f"⚠️ Folder not found: {folder_path}")
+    loop = asyncio.get_running_loop()
+    file_path = f"{FOLDER_PATH}/{topic_id}.json"
+
+    # Check cache
+    cache_entry = _discourse_cache.get(file_path)
+    if cache_entry:
+        age = datetime.now() - cache_entry["timestamp"]
+        if age < CACHE_TTL:
+            logger.info(f"Using cached discourse for topic {topic_id} (age={age})")
+            return cache_entry["data"]
+
+    try:
+        logger.info(f"Fetching discourse {topic_id} from Dropbox: {file_path}")
+        metadata, res = await loop.run_in_executor(None, dbx.files_download, file_path)
+        data = res.content.decode("utf-8")
+        discourse = json.loads(data)
+
+        # Update cache
+        _discourse_cache[file_path] = {"timestamp": datetime.now(), "data": discourse}
+        return discourse
+
+    except dropbox.exceptions.HttpError as e:
+        logger.error(f"Dropbox file not found: {file_path}", exc_info=e)
         return None
-
-    for filename in os.listdir(folder_path):
-        if filename.lower().endswith(".json"):
-            file_path = os.path.join(folder_path, filename)
-            try:
-                with open(file_path, "r", encoding="utf-8") as f:
-                    data = json.load(f)
-                    if data.get("id") == topic_id:
-                        return data
-            except Exception as e:
-                print(f"❌ Error loading {filename}: {e}")
-
-    return None
+    except Exception as e:
+        logger.error(f"Error fetching discourse {topic_id}", exc_info=e)
+        # fallback to cached data if available
+        if cache_entry:
+            logger.warning(f"Returning stale cached discourse for topic {topic_id}")
+            return cache_entry["data"]
+        return None

server.py

@@ -1,8 +1,7 @@
 # server.py
-import json
 import random
 import traceback
-from typing import List, Optional
+from typing import  Optional
 import uuid
 from fastapi import APIRouter, HTTPException, Request, Query
 from fastapi.responses import JSONResponse
@@ -15,7 +14,7 @@ from metadata import MetadataWhereClause
 from modules.audio.model import AudioRequest, AudioType
 from modules.audio.service import svc_get_audio_urls, svc_get_indices_with_audio
 from modules.config.categories import get_scripture_categories
-from modules.dropbox.discources import fetch_all_discourses, get_discourse_by_id, get_discourse_summaries
+from modules.dropbox.discources import get_discourse_by_id, get_discourse_summaries
 from modules.languages.get_v2 import handle_fetch_languages_v2
 from modules.quiz.answer_validator import validate_answer
 from modules.quiz.models import Question
@@ -567,7 +566,7 @@ def route_get_donation_product_ids(include_tests: bool = False):
     return products
 
 @router.get("/discourse/list")
-def get_all_discourses(
+async def get_all_discourses(
     page: int = Query(1, ge=1, description="Page number (1-indexed)"),
     per_page: int = Query(10, ge=1, le=100, description="Number of items per page")
 ):
@@ -578,15 +577,15 @@ def get_all_discourses(
       - topic_name
       - thumbnail_url
     """
-    result = get_discourse_summaries(page=page, per_page=per_page)
+    result = await get_discourse_summaries(page=page, per_page=per_page)
     return result
 
 @router.get("/discourse/find/{topic_id}")
-def get_discourse_detail(topic_id: int):
+async def get_discourse_detail(topic_id: int):
     """
     Returns the full details of a discourse topic by its unique ID.
     """
-    topic = get_discourse_by_id(topic_id)
+    topic = await get_discourse_by_id(topic_id)
     if not topic:
         raise HTTPException(status_code=404, detail="Discourse topic not found")
     return topic