Upload folder using huggingface_hub

config.py

@@ -14,6 +14,11 @@ class SanatanConfig:
             "title": "Sri Vishnu Puranam",
             "output_dir": "./output/vishnu_puranam",
             "collection_name": "vishnu_puranam",
+            "metadata_fields" : [
+                {"name": "file", "datatype": "str"}, 
+                {"name": "num_chars", "datatype": "str"}, 
+                {"name": "page", "datatype": "int"}, 
+            ],
             "pdf_path": "./data/vishnu_puranam.pdf",
             "source": "https://dn720005.ca.archive.org/0/items/vishnu-purana-sanskrit-english-ocr/VISHNU-PURANA-Sanskrit-English-OCR.pdf",
             "language": "san+eng",
@@ -35,6 +40,11 @@ class SanatanConfig:
             "title": "Shukla Yajur Vedam",
             "output_dir": "./output/shukla_yajur_vedam",
             "collection_name": "shukla_yajur_vedam",
+            "metadata_fields" : [
+                {"name": "file", "datatype": "str"}, 
+                {"name": "num_chars", "datatype": "str"}, 
+                {"name": "page", "datatype": "int"}, 
+            ],
             "pdf_path": "./data/shukla-yajur-veda.pdf",
             "source": "https://www.thearyasamaj.org/uploads/book/2014/04/R1sSjG_eLb_sub_406_yajurveda.pdf",
             "language": "san+eng",
@@ -54,6 +64,11 @@ class SanatanConfig:
             "title": "Bhagavat Gita",
             "output_dir": "./output/bhagavat_gita",
             "collection_name": "bhagavat_gita",
+            "metadata_fields" : [
+                {"name": "file", "datatype": "str"}, 
+                {"name": "num_chars", "datatype": "str"}, 
+                {"name": "page", "datatype": "int"}, 
+            ],
             "pdf_path": "./data/bhagavat_gita.pdf",
             "source": "https://dn790006.ca.archive.org/0/items/in.gov.ignca.279/279_text.pdf",
             "language": "san+eng",
@@ -77,6 +92,11 @@ class SanatanConfig:
             "title": "Valmiki Ramayanam",
             "output_dir": "./output/valmiki_ramayanam",
             "collection_name": "valmiki_ramayanam",
+            "metadata_fields" : [
+                {"name": "file", "datatype": "str"}, 
+                {"name": "num_chars", "datatype": "str"}, 
+                {"name": "page", "datatype": "int"}, 
+            ],
             "pdf_path": "./data/valmiki_ramayanam.pdf",
             "source": "https://ia800509.us.archive.org/28/items/valmiki-ramayana-gita-press-english/Valmiki%20Ramayana%20Gita%20Press%20English.pdf",
             "language": "san+eng",
@@ -100,6 +120,14 @@ class SanatanConfig:
             "title": "Vishnu Sahasranamam",
             "output_dir": "./output/vishnu_sahasranamam",
             "collection_name": "vishnu_sahasranamam",
+            "metadata_fields" : [
+                {"name": "chapter", "datatype": "str"}, 
+                {"name": "page_number", "datatype": "int"}, 
+                {"name": "sanskrit", "datatype": "str"}, 
+                {"name": "translation", "datatype": "str"}, 
+                {"name": "transliteration", "datatype": "str"}, 
+                {"name": "verse", "datatype": "int"}, 
+            ],
             "pdf_path": "./data/vishnu_sahasranamam.pdf",
             "source": "https://www.swami-krishnananda.org/vishnu/Sri_Vishnu_Sahasranama_Stotram.pdf",
             "language": "san+eng",
@@ -115,6 +143,18 @@ class SanatanConfig:
             "title": "4000 Divya Prabandham",
             "output_dir": "./output/divya_prabandham",
             "collection_name": "divya_prabandham",
+            "metadata_fields" : [
+                {"name": "prabandham", "datatype": "str", "description" : "contains the prabandham code. e.g. `TPL` for `Thiruppallandu`"}, 
+                {"name": "prabandham_name", "datatype": "str", "description" : "contains the prabandham name. e.g. `Thiruppallandu`"}, 
+                {"name": "azhwar_name", "datatype": "str", "description" : "contains the azhwar name. e.g. `Thirumangai Azhwar`"}, 
+                {"name": "divya_desams", "datatype": "str", "description" : "comma separated list of divya desams. e.g. Thiruneermalai,Thiruvallikkeni"}, 
+                {"name": "html_url", "datatype": "str", "description" : "Reference link for the source"}, 
+                {"name": "pasuram_en", "datatype": "str", "description" : "Transliteration of pasuram in english"}, 
+                {"name": "pasuram_ta", "datatype": "str", "description" : "Pasuram lyrics in tamil"}, 
+                {"name": "title", "datatype": "str", "description" : "Title of this pasuram. for example `TVM 1.8.3`"}, 
+                {"name": "verse", "datatype": "int", "description" : "Verse number or pasuram number."}, 
+                {"name": "wbw_ta", "datatype": "str", "description" : "Word by word meaning in tamil."}, 
+            ],
             "pdf_path": "./data/divya_prabandham.pdf",
             "source": "https://uveda.org",
             "language": "tam",
@@ -136,6 +176,11 @@ class SanatanConfig:
             "title": "Bhagavatha Puranam",
             "output_dir": "./output/bhagavata_purana",
             "collection_name": "bhagavata_purana",
+            "metadata_fields" : [
+                {"name": "file", "datatype": "str"}, 
+                {"name": "num_chars", "datatype": "str"}, 
+                {"name": "page", "datatype": "int"}, 
+            ],
             "pdf_path": "./data/bhagavata_purana.pdf",
             "source": "https://dn790003.ca.archive.org/0/items/bhagavatapuranagitapress_201907/Bhagavata%20Purana%20-%20Gita%20Press_text.pdf",
             "language": "san+eng",
@@ -150,6 +195,14 @@ class SanatanConfig:
             "title": "Kamba Ramayanam (English)",
             "output_dir": "./output/kamba_ramayanam",
             "collection_name": "kamba_ramayanam_en",
+            "metadata_fields" : [
+                {"name": "file", "datatype": "str", "description" : "The name of the Kandam or the chapter."}, 
+                {"name": "padalam_en", "datatype": "str", "description" : "The name of the Padalam (Episode) in English."}, 
+                {"name": "padalam_ta", "datatype": "str", "description" : "The name of the Padalam (Episode) in Tamil."}, 
+                {"name": "page", "datatype": "int"}, 
+                {"name": "verse_number", "datatype": "int"}, 
+            ],
+
             "pdf_path": "./data/kamba_ramayanam.pdf",
             "source": "https://www.hindupedia.com/images/1/13/Kamba_Ramayanam_I.pdf",
             "language": "tam",
@@ -173,6 +226,10 @@ class SanatanConfig:
             "title": "Kamba Ramayanam (Tamil)",
             "output_dir": "./output/kamba_ramayanam",
             "collection_name": "kamba_ramayanam",
+            "metadata_fields" : [
+                {"name": "chunk_index", "datatype": "int", "description" : "The index of the chunk"}, 
+                {"name": "filename", "datatype": "str", "description" : "The name of the file."}, 
+            ],
             "pdf_path": "./data/kamba_ramayanam.pdf",
             "source": "https://archive.org/details/vrajeshkumar_gmail_061/01-%E0%AE%AA%E0%AE%BE%E0%AE%B2%20%E0%AE%95%E0%AE%BE%E0%AE%A3%E0%AF%8D%E0%AE%9F%E0%AE%AE%E0%AF%8D/page/n15/mode/2up",
             "language": "tam",

tools.py

@@ -1,3 +1,4 @@
+import json
 from langchain.agents import Tool
 from langchain_core.tools import StructuredTool
 
@@ -29,11 +30,13 @@ tool_search_db_by_metadata = StructuredTool.from_function(
         f"The collection_name must be one of: {', '.join(allowed_collections)}."
         " Use this to find relevant scripture verses or explanations."
         "if the user asks for a specific azhwar, use the `tool_get_standardized_azhwar_names` tool to get the standard name first and then pass to this tool to filter pasurams based on azhwar_name."
-        "be aware that verse numbers are sometimes stored as strings and sometimes as mumbers, so if str search does not yield results, try passing in the metadata_value as a number instead"
-        "in the context of divya_prabandham, the verse/pasuram number is stored in metadata as the field `verse` and it is  stored as an int."
-        "in the context of sahasranamam, the verse/pasuram number is stored in metadata as the field `verse` and it is  stored as an int."
-        "in the context of kamba_ramayanam, the verse number is stored in metadata as the field `verse_number` and it is  stored as a string datatype."
-        "for other scriptures, the verse number is stored either as `verse` or `verse_number` fields and it can be either str or int so check for both whichever yields results."
+        "if the user asks for a specific prabandham name, use the `tool_get_standardized_azhwar_names` tool to get the standard prabandham name first and then pass to this tool to filter pasurams based on prabandham_name."
+        f"use this configuration for reference :\n{json.dumps(SanatanConfig.scriptures, indent=1)}\n"
+        # "be aware that verse numbers are sometimes stored as strings and sometimes as mumbers, so if str search does not yield results, try passing in the metadata_value as a number instead"
+        # "in the context of divya_prabandham, the verse/pasuram number is stored in metadata as the field `verse` and it is  stored as an int."
+        # "in the context of sahasranamam, the verse/pasuram number is stored in metadata as the field `verse` and it is  stored as an int."
+        # "in the context of kamba_ramayanam, the verse number is stored in metadata as the field `verse_number` and it is  stored as a string datatype."
+        # "for other scriptures, the verse number is stored either as `verse` or `verse_number` fields and it can be either str or int so check for both whichever yields results."
     ),
 )