Upload folder using huggingface_hub

config.py

@@ -352,6 +352,15 @@ class SanatanConfig:
             "output_dir": "./output/kamba_ramayanam",
             "collection_name": "kamba_ramayanam_en",
             "unit": "verse",
+            "unit_field": "verse_number",
+            "field_mapping": {
+                "chapter_name": lambda doc: f"{doc.get('kandam','')}",
+                "title": lambda doc: f"{doc.get('padalam_ta','')} - {doc.get('padalam_en','')}".strip(),
+                "author": lambda doc: "Kamban",
+                "unit_index": "verse_number",
+                "verse": lambda doc: int(doc.get("verse_number", "0")),
+                "relative_path": lambda doc: f"{doc.get('padalam_ta','')} - {doc.get('padalam_en','')}".strip(),
+            },
             "metadata_fields": [
                 {
                     "name": "kandam",
@@ -668,7 +677,6 @@ class SanatanConfig:
         ):
             canonical_doc["text"] = canonical_doc["document"]
             canonical_doc["document"] = "-"
-        canonical_doc["verse"] = resolve_field(
-            config.get("unit_field", config.get("unit"))
-        )
+        verse = resolve_field(config.get("unit_field", config.get("unit")))
+        canonical_doc["verse"] = int(verse) if verse else 0
         return canonical_doc

db.py

@@ -112,7 +112,7 @@ class SanatanDatabase:
                 n_results=n_results,
             )
 
-    def fetch_document_by_index(self, collection_name: str, index: int, unit_name : str):
+    def fetch_document_by_index(self, collection_name: str, index: int, unit_name: str):
         """
         Fetch one document at a time from a ChromaDB collection using pagination (index = 0-based).
 
@@ -133,18 +133,11 @@ class SanatanDatabase:
         collection = self.chroma_client.get_or_create_collection(name=collection_name)
 
         try:
-            # show a sample data record
-            response = collection.get(
-                limit=2,
-                # offset=index,  # pagination via offset
-                include=["metadatas", "documents"],
-            )
-            print(response)
             response = collection.get(
                 limit=1,
                 # offset=index,  # pagination via offset
                 include=["metadatas", "documents"],
-                where={unit_name: index}
+                where={"$or": [{unit_name: index}, {unit_name: str(index)}]},
             )
         except Exception as e:
             logger.error("Error fetching document: %s", e)
@@ -158,11 +151,19 @@ class SanatanDatabase:
             result = {"document": documents[0]}
             if metadatas:
                 result.update(metadatas[0])
+            print("raw data = ", result)
             return result
         else:
             print("No data available")
-            return {"error": "No data available."}
+            # show a sample data record
+            response = collection.get(
+                limit=2,
+                # offset=index,  # pagination via offset
+                include=["metadatas", "documents"],
+            )
+            print("sample data : ",response)
 
+            return {"error": "No data available."}
 
     def search_semantic(
         self,