Upload folder using huggingface_hub

config.py

@@ -1,6 +1,10 @@
 from metadata import MetadataWhereClause
 from typing import List, Dict
 
+from modules.scripture_helpers.bhagavat_gita_helper import (
+    get_bhagavat_gita_chapters,
+    get_chapter_order_from_bhagavat_gita,
+)
 from modules.scripture_helpers.kamba_ramayanam_helper import (
     get_list_of_kandams,
     get_list_of_padalams,
@@ -101,19 +105,54 @@ class SanatanConfig:
             "output_dir": "./output/bhagavat_gita",
             "collection_name": "bhagavat_gita_openai",
             "collection_embedding_fn": "openai",
-            "unit": "page",
+            "unit": "verse",
+            "unit_field": "verse_number",
+            "chapter_order": lambda: get_chapter_order_from_bhagavat_gita(),
+            "field_mapping": {
+                "text": "sanskrit",
+                "title": "verse_title",
+                "unit_index": "relative_verse_number",
+                "transliteration": "transliteration",
+                "word_by_word_native": "word_by_word_meaning",
+                "translation": "translation",
+                "author": lambda doc: f"Maharishi Vedvyas Ji",
+                "chapter_name": "chapter_title",
+                "relative_path": lambda doc: f"{doc.get("chapter_title","")}-{doc.get("relative_verse_number","")}",
+            },
             "metadata_fields": [
                 {
-                    "name": "file",
-                    "label": "File Name",
+                    "name": "chapter",
                     "datatype": "str",
-                    "description": "name of the file from which the information was extracted",
+                    "label": "Chapter Name",
+                    "description": "Name of the Chapter",
+                    "show_as_filter": True,
+                    "component": "dropdown",
+                    "lov": lambda: get_bhagavat_gita_chapters(),
+                    "is_unique": True,
                 },
                 {
-                    "name": "page",
+                    "name": "sanskrit",
+                    "label": "Lyrics in sanskrit",
+                    "datatype": "str",
+                    "description": "The original sloka in sanskrit.",
+                },
+                {
+                    "name": "translation",
+                    "label": "English Translation",
+                    "datatype": "str",
+                    "description": "The english translation.",
+                },
+                {
+                    "name": "transliteration",
+                    "label": "English Transliteration",
+                    "datatype": "str",
+                    "description": "The english transliteration.",
+                },
+                {
+                    "name": "relative_verse_number",
+                    "label": "Relative Verse Number",
                     "datatype": "int",
-                    "label": "Page Number",
-                    "description": "Page number from the source",
+                    "description": "The relative verse number of the sloka inside the chapter.",
                     "show_as_filter": True,
                     "is_unique": True,
                 },
@@ -192,7 +231,7 @@ class SanatanConfig:
             "collection_name": "vishnu_sahasranamam_openai",
             "collection_embedding_fn": "openai",
             "unit": "verse",
-            "chapter_order" : lambda: get_chapter_order_from_sahasranamam(), 
+            "chapter_order": lambda: get_chapter_order_from_sahasranamam(),
             "field_mapping": {
                 "text": "sanskrit",
                 "title": lambda doc: f"Verse {doc.get('verse','')}",
@@ -200,7 +239,7 @@ class SanatanConfig:
                 "transliteration": "transliteration",
                 "word_by_word_native": "translation",
                 "author": lambda doc: f"Sage Vyasa",
-                "chapter_name" : "chapter",
+                "chapter_name": "chapter",
                 "relative_path": lambda doc: f"Vishnu Sahasranamam-{doc.get("chapter","")}-{doc.get("verse","")}",
             },
             "metadata_fields": [
@@ -791,9 +830,11 @@ class SanatanConfig:
             embedding_fn = scripture["collection_embedding_fn"]  # overridden in config
         return embedding_fn
 
-    def remove_callables(self,obj):
+    def remove_callables(self, obj):
         if isinstance(obj, dict):
-            return {k: self.remove_callables(v) for k, v in obj.items() if not callable(v)}
+            return {
+                k: self.remove_callables(v) for k, v in obj.items() if not callable(v)
+            }
         elif isinstance(obj, list):
             return [self.remove_callables(v) for v in obj if not callable(v)]
         else:
@@ -825,6 +866,7 @@ class SanatanConfig:
             "unit",
             "unit_index",
             "word_by_word_native",
+            "translation",
             "transliteration",
             "reference_link",
             "author",

copy_chromadb.py

@@ -18,6 +18,11 @@ db_config = {
         "source_collection_name": "vishnu_sahasranamam",
         "destination_collection_name": "vishnu_sahasranamam_openai",
     },
+    "bhagavat_gita": {
+        "source_db_path": "../bhagavat_gita_chat/chromadb_store",
+        "source_collection_name": "bhagavat_gita",
+        "destination_collection_name": "bhagavat_gita_openai",
+    },
 }
 
 parser = argparse.ArgumentParser(description="My app with database parameter")

modules/scripture_helpers/bhagavat_gita_helper.py

@@ -0,0 +1,18 @@
+import json
+
+def get_chapter_order_from_bhagavat_gita():
+    chapters = []
+    with open("./data/bhagavat_gita_chapters.json", "r", encoding="utf-8") as f:
+        chapters = json.load(f)
+
+    return {chapter["chapter_title"]: chapter["chapter_number"] for chapter in chapters}
+
+def get_bhagavat_gita_chapters() -> list[str]:
+    """
+    Get a list of chapter names from bhagavat_gita
+    """
+    chapters = []
+    with open("./data/bhagavat_gita_chapters.json", "r", encoding="utf-8") as f:
+        chapters = json.load(f)
+
+    return sorted(set([chapter["chapter_title"] for chapter in chapters]))