Upload folder using huggingface_hub

app.py

@@ -58,6 +58,8 @@ def init():
         )
         downloader.unzip(zip_path, extract_to="./")
 
+    # add global index
+    SanatanDatabase().build_global_index_for_all_scriptures()
 
 def render_message_with_tooltip(content: str, max_chars=200):
     short = escape(content[:max_chars]) + ("…" if len(content) > max_chars else "")

config.py

@@ -1,11 +1,15 @@
 from metadata import MetadataWhereClause
 from typing import List, Dict
 
-from modules.kamba_ramayanam_helper import (
+from modules.scripture_helpers.kamba_ramayanam_helper import (
     get_list_of_kandams,
     get_list_of_padalams,
     get_list_of_padalams_in_tamil,
 )
+from modules.scripture_helpers.vishnu_sahasranamam_helper import (
+    get_chapter_order_from_sahasranamam,
+    get_chapters_from_sahasranamam,
+)
 import nalayiram_helper
 
 
@@ -32,6 +36,7 @@ class SanatanConfig:
                     "label": "Page Number",
                     "description": "Page number from the source",
                     "show_as_filter": True,
+                    "is_unique": True,
                 },
             ],
             "pdf_path": "./data/vishnu_puranam.pdf",
@@ -72,6 +77,7 @@ class SanatanConfig:
                     "label": "Page Number",
                     "description": "Page number from the source",
                     "show_as_filter": True,
+                    "is_unique": True,
                 },
             ],
             "pdf_path": "./data/shukla-yajur-veda.pdf",
@@ -109,6 +115,7 @@ class SanatanConfig:
                     "label": "Page Number",
                     "description": "Page number from the source",
                     "show_as_filter": True,
+                    "is_unique": True,
                 },
             ],
             "pdf_path": "./data/bhagavat_gita.pdf",
@@ -152,6 +159,7 @@ class SanatanConfig:
                     "label": "Page Number",
                     "description": "Page number from the source",
                     "show_as_filter": True,
+                    "is_unique": True,
                 },
             ],
             "pdf_path": "./data/valmiki_ramayanam.pdf",
@@ -184,6 +192,7 @@ class SanatanConfig:
             "collection_name": "vishnu_sahasranamam_openai",
             "collection_embedding_fn": "openai",
             "unit": "verse",
+            "chapter_order" : lambda: get_chapter_order_from_sahasranamam(), 
             "field_mapping": {
                 "text": "sanskrit",
                 "title": lambda doc: f"Verse {doc.get('verse','')}",
@@ -191,6 +200,7 @@ class SanatanConfig:
                 "transliteration": "transliteration",
                 "word_by_word_native": "translation",
                 "author": lambda doc: f"Sage Vyasa",
+                "chapter_name" : "chapter",
                 "relative_path": lambda doc: f"Vishnu Sahasranamam-{doc.get("chapter","")}-{doc.get("verse","")}",
             },
             "metadata_fields": [
@@ -199,6 +209,10 @@ class SanatanConfig:
                     "datatype": "str",
                     "label": "Chapter Name",
                     "description": "Name of the Chapter",
+                    "show_as_filter": True,
+                    "component": "dropdown",
+                    "lov": lambda: get_chapters_from_sahasranamam(),
+                    "is_unique": True,
                 },
                 {
                     "name": "page_number",
@@ -211,28 +225,26 @@ class SanatanConfig:
                     "label": "Lyrics in sanskrit",
                     "datatype": "str",
                     "description": "The original sloka in sanskrit.",
-                    "show_as_filter": True,
                 },
                 {
                     "name": "translation",
                     "label": "English Translation",
                     "datatype": "str",
                     "description": "The english translation.",
-                    "show_as_filter": True,
                 },
                 {
                     "name": "transliteration",
                     "label": "English Transliteration",
                     "datatype": "str",
                     "description": "The english transliteration.",
-                    "show_as_filter": True,
                 },
                 {
                     "name": "verse",
-                    "labek": "Verse Number",
+                    "label": "Verse Number",
                     "datatype": "int",
                     "description": "The verse number of the sloka.",
                     "show_as_filter": True,
+                    "is_unique": True,
                 },
             ],
             "pdf_path": "./data/vishnu_sahasranamam.pdf",
@@ -427,6 +439,7 @@ class SanatanConfig:
                     "label": "Page Number",
                     "description": "Page number from the source",
                     "show_as_filter": True,
+                    "is_unique": True,
                 },
             ],
             "pdf_path": "./data/bhagavata_purana.pdf",
@@ -495,6 +508,7 @@ class SanatanConfig:
                     "label": "Verse Number",
                     "description": "Verse Number",
                     "show_as_filter": True,
+                    "is_unique": True,
                 },
             ],
             "pdf_path": "./data/kamba_ramayanam.pdf",
@@ -538,6 +552,7 @@ class SanatanConfig:
                     "datatype": "int",
                     "description": "The index of the chunk",
                     "show_as_filter": True,
+                    "is_unique": True,
                 },
                 {
                     "name": "filename",
@@ -588,6 +603,7 @@ class SanatanConfig:
                     "datatype": "int",
                     "description": "The index of the sloka or verse",
                     "show_as_filter": True,
+                    "is_unique": True,
                 },
                 {
                     "name": "meaning_short",
@@ -631,6 +647,7 @@ class SanatanConfig:
                     "datatype": "int",
                     "description": "The index of the sloka or verse",
                     "show_as_filter": True,
+                    "is_unique": True,
                 },
                 {
                     "name": "meaning_short",
@@ -686,6 +703,7 @@ class SanatanConfig:
                     "datatype": "str",
                     "description": "The video id as in YouTube",
                     "show_as_filter": True,
+                    "is_unique": True,
                 },
                 {
                     "name": "video_title",
@@ -792,6 +810,8 @@ class SanatanConfig:
         Only allows keys from the allowed canonical fields list.
         """
         allowed_keys = {
+            "_global_index",
+            "id",
             "verse",
             "text",
             "title",
@@ -843,6 +863,8 @@ class SanatanConfig:
             canonical_doc["document"] = "-"
         verse = resolve_field(config.get("unit_field", config.get("unit")))
         canonical_doc["verse"] = int(verse) if verse else 0
+        canonical_doc["id"] = resolve_field("id")
+        canonical_doc["_global_index"] = resolve_field("_global_index")
         return canonical_doc
 
     def get_collection_name(self, scripture_name):

db.py

@@ -1,4 +1,5 @@
-import json
+import pandas as pd
+import numpy as np
 import random
 from typing import Literal
 import chromadb
@@ -51,6 +52,7 @@ class SanatanDatabase:
         )
         collection = self.chroma_client.get_or_create_collection(name=collection_name)
         data = collection.get(
+            include=["metadatas", "documents"],
             where=(
                 metadata_where_clause.to_chroma_where()
                 if metadata_where_clause is not None
@@ -74,6 +76,39 @@ class SanatanDatabase:
             metadatas=[metas[i] for i in indices],
         )
 
+    def fetch_first_match(
+        self,
+        collection_name: str,
+        metadata_where_clause: MetadataWhereClause = None
+    ):
+        """This version is created to support the browse module"""
+        logger.info(
+            "getting first matching verses from [%s] | metadata_where_clause = %s",
+            collection_name,
+            metadata_where_clause,
+        )
+        collection = self.chroma_client.get_or_create_collection(name=collection_name)
+        data = collection.get(
+            limit=1, #hardcoded to 1 by design
+            include=["metadatas", "documents"],
+            where=(
+                metadata_where_clause.to_chroma_where()
+                if metadata_where_clause is not None
+                else None
+            )
+        )
+        docs = data["documents"]  # list of all verse texts
+        ids = data["ids"]
+        metas = data["metadatas"]
+
+        if not docs:
+            logger.warning("No data found! - data=%s", data)
+            return chromadb.GetResult(ids=[], documents=[], metadatas=[])
+
+        # pick k random indices
+
+        return data
+
     def search(
         self,
         collection_name: str,
@@ -112,7 +147,7 @@ class SanatanDatabase:
                 n_results=n_results,
             )
 
-    def fetch_document_by_index(self, collection_name: str, index: int, unit_name: str):
+    def fetch_document_by_index(self, collection_name: str, index: int):
         """
         Fetch one document at a time from a ChromaDB collection using pagination (index = 0-based).
 
@@ -129,7 +164,7 @@ class SanatanDatabase:
             }
             Or a dict with "error" key if something went wrong.
         """
-        logger.info("fetching %s #%d from [%s]", unit_name, index, collection_name)
+        logger.info("fetching index %d from [%s]", index, collection_name)
         collection = self.chroma_client.get_or_create_collection(name=collection_name)
 
         try:
@@ -137,31 +172,27 @@ class SanatanDatabase:
                 limit=1,
                 # offset=index,  # pagination via offset
                 include=["metadatas", "documents"],
-                where={"$or": [{unit_name: index}, {unit_name: str(index)}]},
+                where={"_global_index": index},
             )
         except Exception as e:
-            logger.error("Error fetching document: %s", e)
+            logger.error("Error fetching document: %s", e, exc_info=True)
             return {"error": f"There was an error fetching the document: {str(e)}"}
 
         documents = response.get("documents", [])
         metadatas = response.get("metadatas", [])
+        ids = response.get("ids", [])
 
         if documents:
             # merge document text with metadata
             result = {"document": documents[0]}
             if metadatas:
                 result.update(metadatas[0])
-            # print("raw data = ", result)
+            if ids:
+                result["id"] = ids[0]
+            print("raw data = ", result)
             return result
         else:
             print("No data available")
-            if index == 1:
-                # there should be atleast one row in the collection?
-                # add index
-                self.add_unit_index_to_collection(
-                    collection_name=collection_name, unit_field=unit_name
-                )
-
             # show a sample data record
             response1 = collection.get(
                 limit=2,
@@ -449,7 +480,10 @@ class SanatanDatabase:
         print(
             f"✅ Finished adding {unit_field} to {unit_counter-1} documents in {collection_name}."
         )
-    def get_list_of_values(self, collection_name: str, metadata_field_name: str) -> list:
+
+    def get_list_of_values(
+        self, collection_name: str, metadata_field_name: str
+    ) -> list:
         """
         Returns the unique values for a given metadata field in a collection.
         """
@@ -477,3 +511,146 @@ class SanatanDatabase:
                             values.add(md[metadata_field_name])
 
         return sorted(list(values))
+
+    def build_global_index_for_all_scriptures(self, force: bool = False):
+        import pandas as pd
+        import numpy as np
+
+        logger.info("build_global_index_for_all_scriptures: started")
+        config = SanatanConfig()
+
+        for scripture in config.scriptures:
+            scripture_name = scripture["name"]
+            chapter_order = scripture.get("chapter_order", None)
+            # if scripture_name != "vishnu_sahasranamam":
+            #     continue
+            logger.info(
+                "build_global_index_for_all_scriptures:%s: Processing", scripture_name
+            )
+            collection_name = scripture["collection_name"]
+            collection = self.chroma_client.get_or_create_collection(
+                name=collection_name
+            )
+            metadata_fields = scripture.get("metadata_fields", [])
+
+            # Get metadata field names marked as unique
+            unique_fields = [f["name"] for f in metadata_fields if f.get("is_unique")]
+            if not unique_fields:
+                if metadata_fields:
+                    unique_fields = [metadata_fields[0]["name"]]
+                else:
+                    logger.warning(
+                        f"No metadata fields defined for {collection_name}, skipping"
+                    )
+                    continue
+
+            logger.info(
+                "build_global_index_for_all_scriptures:%s:unique fields: %s",
+                scripture_name,
+                unique_fields,
+            )
+
+            # Build chapter_order mapping if defined
+            chapter_order_mapping = {}
+            for field in metadata_fields:
+                if callable(chapter_order):
+                    chapter_order_mapping = chapter_order()
+            logger.info(
+                "build_global_index_for_all_scriptures:%s:chapter_order_mapping: %s",
+                scripture_name,
+                chapter_order_mapping,
+            )
+
+            # Fetch all records (keep embeddings for upsert)
+            try:
+                results = collection.get(
+                    include=["metadatas", "documents", "embeddings"]
+                )
+            except Exception as e:
+                logger.error(
+                    "build_global_index_for_all_scriptures:%s Error getting data from chromadb",
+                    scripture_name,
+                    exc_info=True,
+                )
+                continue
+
+            ids = results["ids"]
+            metadatas = results["metadatas"]
+            documents = results["documents"]
+            embeddings = results.get("embeddings", [None] * len(ids))
+
+            if not force and metadatas and "_global_index" in metadatas[0]:
+                logger.warning(
+                    "build_global_index_for_all_scriptures:%s: global index already available. skipping collection",
+                    scripture_name,
+                )
+                continue
+
+            # Create a DataFrame for metadata sorting
+            df = pd.DataFrame(metadatas)
+            df["_id"] = ids
+            df["_doc"] = documents
+
+            # Add sortable columns for each unique field
+            for field_name in unique_fields:
+                if field_name.lower() == "chapter" and chapter_order_mapping:
+                    # Map chapter names to their defined order
+                    df["_sort_" + field_name] = (
+                        df[field_name].map(chapter_order_mapping).fillna(np.inf)
+                    )
+                else:
+                    # Try numeric, fallback to string lowercase
+                    def parse_val(v):
+                        if v is None:
+                            return float("inf")
+                        if isinstance(v, int):
+                            return v
+                        if isinstance(v, str):
+                            v = v.strip()
+                            return int(v) if v.isdigit() else v.lower()
+                        return str(v)
+
+                    df["_sort_" + field_name] = df[field_name].apply(parse_val)
+
+            sort_cols = ["_sort_" + f for f in unique_fields]
+            df = df.sort_values(by=sort_cols, kind="stable").reset_index(drop=True)
+
+            # Assign global index
+            df["_global_index"] = range(1, len(df) + 1)
+
+            logger.info(
+                "build_global_index_for_all_scriptures:%s: updating database",
+                scripture_name,
+            )
+
+            # Batch upsert
+            BATCH_SIZE = 5000  # safely below max batch size
+            for i in range(0, len(df), BATCH_SIZE):
+                batch_df = df.iloc[i : i + BATCH_SIZE]
+                batch_ids = batch_df["_id"].tolist()
+                batch_docs = batch_df["_doc"].tolist()
+                batch_metas = [
+                    {k: record[k] for k in metadatas[0].keys() if k in record}
+                    | {"_global_index": record["_global_index"]}
+                    for record in batch_df.to_dict(orient="records")
+                ]
+                # Use original metadata keys for upsert
+                batch_metas = [
+                    {k: record[k] for k in metadatas[0].keys() if k in record}
+                    | {"_global_index": record["_global_index"]}
+                    for record in batch_df.to_dict(orient="records")
+                ]
+                batch_embeds = [embeddings[idx] for idx in batch_df.index]
+
+                collection.upsert(
+                    ids=batch_ids,
+                    documents=batch_docs,
+                    metadatas=batch_metas,
+                    embeddings=batch_embeds,
+                )
+
+            logger.info(
+                "build_global_index_for_all_scriptures:%s: ✅ Updated with %d records",
+                scripture_name,
+                len(df),
+            )

main.py

@@ -1,6 +1,7 @@
 from fastapi.responses import RedirectResponse
 import uvicorn
 from fastapi import FastAPI
+from db import SanatanDatabase
 from server import router as mobile_router
 from app import gradio_app  # your Blocks object
 import gradio as gr
@@ -31,4 +32,4 @@ async def log_requests(request: Request, call_next):
     return response
 
 if __name__ == "__main__":
-    uvicorn.run("main:app", host="0.0.0.0", port=7860, reload=True)
+    uvicorn.run("main:app", host="0.0.0.0", port=7860, reload=False)

modules/scripture_helpers/kamba_ramayanam_helper.py

@@ -0,0 +1,158 @@
+def get_list_of_kandams():
+    return sorted([
+        "Yudha Kandam",
+        "Ayodhya Kandam",
+        "Kishkinda Kandam",
+        "Aranya Kandam",
+        "Sundara Kandam",
+        "Kamba Ramayanam- Bala Kandam",
+        "1Bala Kandam",
+    ])
+
+
+def get_list_of_padalams():
+    return sorted([
+        "",
+        "The chapter on moon mountain",
+        "The chapter on rainy season",
+        "The chapter on river",
+        "The chapter on lake at Pambai",
+        "The chapter on death of Sarabanga",
+        "The chapter on floating in oil",
+        "The chapter on fire sacrifice",
+        "The chapter on handing over",
+        "The chapter on killing of Viradha",
+        "The chapter on seeing ganges",
+        "The chapter on Ahalya",
+        "The chapter on picturesque description",
+        "The chapter on the great marriage",
+        "The chapter on going away for search",
+        "The chapter on narration about family of Rama.",
+        "The chapter on dancing after drinking",
+        "The chapter on conspiracy of Kaikeyi",
+        "The chapter on divine incarnation",
+        "The chapter on friendship",
+        "The chapter on the country",
+        "The chapter on Kabandha",
+        "The chapter on governance",
+        "The chapter on Agasthya",
+        "The chapter on looks of MIthila",
+    ])
+
+def get_list_of_padalams_in_tamil():
+    return sorted([
+        "",
+        "Vidai Kodu Padalam",
+        "Kakshi Padalam",
+        "Yethir kol Padalam",
+        "Soorpankai padalam",
+        "Kaarkala Padalam",
+        "Pini Veetu padalam",
+        "Thiru avathara Padalam",
+        "Choodamani padalam",
+        "Ezhuchi Padalam",
+        "Pambai Vaavi padalam",
+        "Poo Koi padalam",
+        "Ravanan Kalam Kaan Padalam",
+        "Dundhubhi padalam",
+        "Arasiyar Padalam",
+        "Varai kakshi padalam",
+        "Makara Kannan Vadhai padalam",
+        "Maya Sithai Padalam",
+        "Angadhan Dhoothu Padalam",
+        "Thadagai vadhai padalam",
+        "Chitrakoota Padalam",
+        "KItkindhai Padalam",
+        "Raman ther yerum Padalam",
+        "Ulaviyar Padalam",
+        "Meekshi Padalam",
+        "Nikumbalai Yaga Padalam",
+        "Mahendra Padalam",
+        "Kula murai kilathu Padalam",
+        "Kaliyattu Padalam",
+        "8,Jambu mali vadhai padalam",
+        "Nattu Padalam",
+        "Nagara Padalam",
+        "Moola Bala Vadhai padalam",
+        "Agathiya Padalam",
+        "Ahaligai padalam",
+        "Oor thedu padalam",
+        "Indirachithu vadhai padalam",
+        "Natpu Kot Padalam",
+        "10Ravanan Vanara thanai kaan Padalam",
+        "Padai Kakshi Padalam",
+        "Velvi Padalam",
+        ".22. Seethai Kalam Kan padalam",
+        "Nagar neengu padalam",
+        "Ravanan ther yeru padalam",
+        "Palli padai padalam",
+        "Anuma Padalam",
+        "Kharan Vadha Padalam",
+        "Vanam pugu Padalam",
+        "Kaikeyi choozhvinai Padalam",
+        "Kalan Kaan Padalam",
+        "Ayomukhi Padalam",
+        "Kumbhakarunan Vadhai Padalam",
+        "Ravananan Vadhai Padalam",
+        "Ani Vakuppu Padalam",
+        "Naada vita Padalam",
+        "Sampathi Padalam",
+        "Karmukha padalam",
+        "Sarabangan pirappu neengu padalam",
+        "Savari pirappu neengu padalam",
+        "Chandra Saila Padalam",
+        "Arasiyal padalam",
+        "Gangai Padalam",
+        "Thailam aatu Padalam",
+        "Hiranyan Vadhai Padalam",
+        "Ravanana Mandira Padalam",
+        "8,Otthu Kelvi Padalam",
+        "Ilangai kaan Padalam",
+        "Athikayan vadhai Padalam",
+        "Ravanan choozhchi Padalam",
+        "Mandhira Padalam",
+        "Maya Janaka Padalam",
+        "Mudal por puri Padalam",
+        "Kolam kaan padalam",
+        "Varunanai vazhi vendu padalam",
+        "Parasurama Padalam",
+        "Vel Yetha Padalam",
+        "Mareechan vadhai padalam",
+        "Naga Pasa Padalam",
+        "Mithilai Kakshi Padalam",
+        "PIlam pugu padalam",
+        "Thirumudi chootu Padalam",
+        "Undattu Padalam",
+        "Kadal kaan Padalam",
+        "23,Maruthu malai Padalam",
+        "Sethu Bandhana Padalam",
+        "Ilankai eriyuttu padalam",
+        "4,Veedanan adaikkala Padalam",
+        "Vali Vadhai Padalam",
+        "AAthu Padalam",
+        "Neer Vilayattu Padalam",
+        "Padai thalaivar Vadhai Padalam",
+        "Kinkarar vadhai padalam",
+        "Kavandhan Padalam",
+        "Pozhil irutha Padalam",
+        "Chadayu kaan Padalam",
+        "Kayadai padalam",
+        "Ilangai kelvi Padalam",
+        "Dandakaranya Padalam",
+        "11,Maguda Banga Padalam",
+        "AAru chel padalam",
+        "Kadal thavu Padalam",
+        "Brahmathira Padalam",
+        "Gangai kaan padalam",
+        "Ravanan Soka Padalam",
+        "Guha Padalam",
+        "Soorpanakai Choozhchi padalam",
+        "Katimana Padalam",
+        "Maramara Padalam",
+        "32,Vanarar kalam kaan Padalam",
+        "14,Thiruvadi thozhutha Padalam",
+        "Thiruvadi Chootu Padalam",
+        "Aaru chel padalam",
+        "Viradhan Vadhai padalam",
+        "Jatayu uyir neetha padalam",
+    ])

modules/scripture_helpers/vishnu_sahasranamam_helper.py

@@ -0,0 +1,13 @@
+def get_chapters_from_sahasranamam():
+    return sorted(
+        ["MEDITATION ON VISHNU", "ATHA DHYĀNAM", "STOTRAM — 1000 NAMES", "PHALASHRUTHI"]
+    )
+
+
+def get_chapter_order_from_sahasranamam():
+    return {
+        "MEDITATION ON VISHNU": 1,
+        "ATHA DHYĀNAM": 2,
+        "STOTRAM — 1000 NAMES": 3,
+        "PHALASHRUTHI": 4,
+    }

server.py

@@ -15,6 +15,11 @@ from metadata import MetadataWhereClause
 from modules.quiz.answer_validator import validate_answer
 from modules.quiz.models import Question
 from modules.quiz.quiz_helper import generate_question
+import logging
+
+logging.basicConfig()
+logger = logging.getLogger(__name__)
+logger.setLevel(logging.INFO)
 
 router = APIRouter()
 
@@ -201,8 +206,9 @@ async def get_scripture(req: ScriptureRequest):
     """
     Return a scripture unit (page or verse, based on config),
     including all metadata fields separately.
+    used for page view to fetch by global index.
     """
-    print("received request to fetch scripture.", req)
+    logger.info("get_scripture: received request to fetch scripture: %s", req)
 
     # find config entry for the scripture
     config = next(
@@ -215,10 +221,10 @@ async def get_scripture(req: ScriptureRequest):
     raw_doc = SanatanDatabase().fetch_document_by_index(
         collection_name=config["collection_name"],
         index=req.unit_index,
-        unit_name=config.get("unit_field", config.get("unit")),
+        # unit_name=config.get("unit_field", config.get("unit")),
     )
 
-    if not raw_doc or isinstance(raw_doc, str):
+    if not raw_doc or isinstance(raw_doc, str) or "error" in raw_doc:
         return {"error": f"No data available for unit {req.unit_index}"}
 
     # canonicalize it
@@ -229,7 +235,6 @@ async def get_scripture(req: ScriptureRequest):
     )
 
     # add unit index & total units (so Flutter can paginate)
-    canonical_doc["unit_index"] = req.unit_index
     canonical_doc["total"] = SanatanDatabase().count(config["collection_name"])
 
     print("canonical_doc = ", canonical_doc)
@@ -275,7 +280,6 @@ async def get_scripture_configs():
 async def search_scripture(
     scripture_name: str,
     filter_obj: Optional[MetadataWhereClause] = None,
-    n_results: int = 1,
 ):
     """
     Search scripture collection with optional filters.
@@ -285,17 +289,23 @@ async def search_scripture(
     """
     try:
         db = SanatanDatabase()
-        results = db.fetch_random_data(
-            collection_name=scripture_name,
+        config = next(
+            (s for s in SanatanConfig().scriptures if s["name"] == scripture_name), None
+        )
+
+        results = db.fetch_first_match(
+            collection_name=config["collection_name"],
             metadata_where_clause=filter_obj,
-            n_results=n_results,
         )
 
         print("results = ", results)
         # Flatten + canonicalize results
         formatted_results = []
         for i in range(len(results["metadatas"])):
+            id = results["ids"][i]
             metadata_doc = results["metadatas"][i]
+            metadata_doc["id"] = id
+            print("metadata_doc = ",metadata_doc)
             document_text = (
                 results["documents"][i] if results.get("documents") else None
             )
@@ -305,7 +315,7 @@ async def search_scripture(
             )
             formatted_results.append(canonical_doc)
 
-        # print("formatted_results = ", formatted_results)
+        print("formatted_results = ", formatted_results)
         return {"results": formatted_results}
 
     except Exception as e:

tests/test_db.py

@@ -36,7 +36,7 @@ if __name__ == "__main__":
                 ),
             ]
         )
-        response = database.search_by_metadata(
+        response = database.search(
             collection_name=collection_name,
             query=query,
             metadata_where_clause=metadata_where_clause,

tests/test_gen_global_index.py

@@ -0,0 +1,12 @@
+import json
+import logging
+
+from db import SanatanDatabase
+from metadata import MetadataFilter, MetadataWhereClause
+
+
+if __name__ == "__main__":
+    logging.basicConfig()
+    collection_name = "divya_prabandham"
+    database = SanatanDatabase()
+    database.build_global_index_for_all_scriptures(force=True)