Upload folder using huggingface_hub

config.py

@@ -14,10 +14,10 @@ class SanatanConfig:
             "title": "Sri Vishnu Puranam",
             "output_dir": "./output/vishnu_puranam",
             "collection_name": "vishnu_puranam",
-            "metadata_fields" : [
-                {"name": "file", "datatype": "str"}, 
-                {"name": "num_chars", "datatype": "str"}, 
-                {"name": "page", "datatype": "int"}, 
+            "metadata_fields": [
+                {"name": "file", "datatype": "str"},
+                {"name": "num_chars", "datatype": "str"},
+                {"name": "page", "datatype": "int"},
             ],
             "pdf_path": "./data/vishnu_puranam.pdf",
             "source": "https://dn720005.ca.archive.org/0/items/vishnu-purana-sanskrit-english-ocr/VISHNU-PURANA-Sanskrit-English-OCR.pdf",
@@ -40,10 +40,10 @@ class SanatanConfig:
             "title": "Shukla Yajur Vedam",
             "output_dir": "./output/shukla_yajur_vedam",
             "collection_name": "shukla_yajur_vedam",
-            "metadata_fields" : [
-                {"name": "file", "datatype": "str"}, 
-                {"name": "num_chars", "datatype": "str"}, 
-                {"name": "page", "datatype": "int"}, 
+            "metadata_fields": [
+                {"name": "file", "datatype": "str"},
+                {"name": "num_chars", "datatype": "str"},
+                {"name": "page", "datatype": "int"},
             ],
             "pdf_path": "./data/shukla-yajur-veda.pdf",
             "source": "https://www.thearyasamaj.org/uploads/book/2014/04/R1sSjG_eLb_sub_406_yajurveda.pdf",
@@ -64,10 +64,10 @@ class SanatanConfig:
             "title": "Bhagavat Gita",
             "output_dir": "./output/bhagavat_gita",
             "collection_name": "bhagavat_gita",
-            "metadata_fields" : [
-                {"name": "file", "datatype": "str"}, 
-                {"name": "num_chars", "datatype": "str"}, 
-                {"name": "page", "datatype": "int"}, 
+            "metadata_fields": [
+                {"name": "file", "datatype": "str"},
+                {"name": "num_chars", "datatype": "str"},
+                {"name": "page", "datatype": "int"},
             ],
             "pdf_path": "./data/bhagavat_gita.pdf",
             "source": "https://dn790006.ca.archive.org/0/items/in.gov.ignca.279/279_text.pdf",
@@ -92,10 +92,10 @@ class SanatanConfig:
             "title": "Valmiki Ramayanam",
             "output_dir": "./output/valmiki_ramayanam",
             "collection_name": "valmiki_ramayanam",
-            "metadata_fields" : [
-                {"name": "file", "datatype": "str"}, 
-                {"name": "num_chars", "datatype": "str"}, 
-                {"name": "page", "datatype": "int"}, 
+            "metadata_fields": [
+                {"name": "file", "datatype": "str"},
+                {"name": "num_chars", "datatype": "str"},
+                {"name": "page", "datatype": "int"},
             ],
             "pdf_path": "./data/valmiki_ramayanam.pdf",
             "source": "https://ia800509.us.archive.org/28/items/valmiki-ramayana-gita-press-english/Valmiki%20Ramayana%20Gita%20Press%20English.pdf",
@@ -120,13 +120,13 @@ class SanatanConfig:
             "title": "Vishnu Sahasranamam",
             "output_dir": "./output/vishnu_sahasranamam",
             "collection_name": "vishnu_sahasranamam",
-            "metadata_fields" : [
-                {"name": "chapter", "datatype": "str"}, 
-                {"name": "page_number", "datatype": "int"}, 
-                {"name": "sanskrit", "datatype": "str"}, 
-                {"name": "translation", "datatype": "str"}, 
-                {"name": "transliteration", "datatype": "str"}, 
-                {"name": "verse", "datatype": "int"}, 
+            "metadata_fields": [
+                {"name": "chapter", "datatype": "str"},
+                {"name": "page_number", "datatype": "int"},
+                {"name": "sanskrit", "datatype": "str"},
+                {"name": "translation", "datatype": "str"},
+                {"name": "transliteration", "datatype": "str"},
+                {"name": "verse", "datatype": "int"},
             ],
             "pdf_path": "./data/vishnu_sahasranamam.pdf",
             "source": "https://www.swami-krishnananda.org/vishnu/Sri_Vishnu_Sahasranama_Stotram.pdf",
@@ -143,17 +143,41 @@ class SanatanConfig:
             "title": "4000 Divya Prabandham",
             "output_dir": "./output/divya_prabandham",
             "collection_name": "divya_prabandham",
-            "metadata_fields" : [
-                {"name": "prabandham", "datatype": "str", "description" : "contains the prabandham code. e.g. `TPL` for `Thiruppallandu`"}, 
-                {"name": "prabandham_name", "datatype": "str", "description" : "contains the prabandham name. e.g. `Thiruppallandu`"}, 
-                {"name": "azhwar_name", "datatype": "str", "description" : "contains the azhwar name. e.g. `Thirumangai Azhwar`"}, 
-                {"name": "divya_desams", "datatype": "str", "description" : "comma separated list of divya desams. e.g. Thiruneermalai,Thiruvallikkeni"}, 
-                {"name": "html_url", "datatype": "str", "description" : "Reference link for the source"}, 
-                {"name": "pasuram_en", "datatype": "str", "description" : "Transliteration of pasuram in english"}, 
-                {"name": "pasuram_ta", "datatype": "str", "description" : "Pasuram lyrics in tamil"}, 
-                {"name": "title", "datatype": "str", "description" : "Title of this pasuram. for example `TVM 1.8.3`"}, 
-                {"name": "verse", "datatype": "int", "description" : "Verse number or pasuram number."}, 
-                {"name": "wbw_ta", "datatype": "str", "description" : "Word by word meaning in tamil."}, 
+            "metadata_fields": [
+                {
+                    "name": "prabandham",
+                    "datatype": "str",
+                    "description": "contains the short prabandham_code. e.g. `TPL` for `Thiruppallandu`",
+                },
+                {
+                    "name": "prabandham_name",
+                    "datatype": "str",
+                    "description": "contains the prabandham name. e.g. `Thiruppallandu`",
+                },
+                {
+                    "name": "azhwar_name",
+                    "datatype": "str",
+                    "description": "contains the azhwar name. e.g. `Thirumangai Azhwar`",
+                },
+                {
+                    "name": "divya_desams",
+                    "datatype": "str",
+                    "description": "comma separated list of divya desams. e.g. Thiruneermalai,Thiruvallikkeni",
+                },
+                # {"name": "html_url", "datatype": "str", "description" : "Reference link for the source"},
+                # {"name": "pasuram_en", "datatype": "str", "description" : "Transliteration of pasuram in english"},
+                # {"name": "pasuram_ta", "datatype": "str", "description" : "Pasuram lyrics in tamil"},
+                {
+                    "name": "title",
+                    "datatype": "str",
+                    "description": "Title of this pasuram. for example `TVM 1.8.3`",
+                },
+                {
+                    "name": "verse",
+                    "datatype": "int",
+                    "description": "Verse number or pasuram number.",
+                },
+                # {"name": "wbw_ta", "datatype": "str", "description" : "Word by word meaning in tamil."},
             ],
             "pdf_path": "./data/divya_prabandham.pdf",
             "source": "https://uveda.org",
@@ -176,10 +200,10 @@ class SanatanConfig:
             "title": "Bhagavatha Puranam",
             "output_dir": "./output/bhagavata_purana",
             "collection_name": "bhagavata_purana",
-            "metadata_fields" : [
-                {"name": "file", "datatype": "str"}, 
-                {"name": "num_chars", "datatype": "str"}, 
-                {"name": "page", "datatype": "int"}, 
+            "metadata_fields": [
+                {"name": "file", "datatype": "str"},
+                {"name": "num_chars", "datatype": "str"},
+                {"name": "page", "datatype": "int"},
             ],
             "pdf_path": "./data/bhagavata_purana.pdf",
             "source": "https://dn790003.ca.archive.org/0/items/bhagavatapuranagitapress_201907/Bhagavata%20Purana%20-%20Gita%20Press_text.pdf",
@@ -195,14 +219,25 @@ class SanatanConfig:
             "title": "Kamba Ramayanam (English)",
             "output_dir": "./output/kamba_ramayanam",
             "collection_name": "kamba_ramayanam_en",
-            "metadata_fields" : [
-                {"name": "file", "datatype": "str", "description" : "The name of the Kandam or the chapter."}, 
-                {"name": "padalam_en", "datatype": "str", "description" : "The name of the Padalam (Episode) in English."}, 
-                {"name": "padalam_ta", "datatype": "str", "description" : "The name of the Padalam (Episode) in Tamil."}, 
-                {"name": "page", "datatype": "int"}, 
-                {"name": "verse_number", "datatype": "int"}, 
+            "metadata_fields": [
+                {
+                    "name": "file",
+                    "datatype": "str",
+                    "description": "The name of the Kandam or the chapter.",
+                },
+                {
+                    "name": "padalam_en",
+                    "datatype": "str",
+                    "description": "The name of the Padalam (Episode) in English.",
+                },
+                {
+                    "name": "padalam_ta",
+                    "datatype": "str",
+                    "description": "The name of the Padalam (Episode) in Tamil.",
+                },
+                {"name": "page", "datatype": "int"},
+                {"name": "verse_number", "datatype": "int"},
             ],
-
             "pdf_path": "./data/kamba_ramayanam.pdf",
             "source": "https://www.hindupedia.com/images/1/13/Kamba_Ramayanam_I.pdf",
             "language": "tam",
@@ -226,9 +261,17 @@ class SanatanConfig:
             "title": "Kamba Ramayanam (Tamil)",
             "output_dir": "./output/kamba_ramayanam",
             "collection_name": "kamba_ramayanam",
-            "metadata_fields" : [
-                {"name": "chunk_index", "datatype": "int", "description" : "The index of the chunk"}, 
-                {"name": "filename", "datatype": "str", "description" : "The name of the file."}, 
+            "metadata_fields": [
+                {
+                    "name": "chunk_index",
+                    "datatype": "int",
+                    "description": "The index of the chunk",
+                },
+                {
+                    "name": "filename",
+                    "datatype": "str",
+                    "description": "The name of the file.",
+                },
             ],
             "pdf_path": "./data/kamba_ramayanam.pdf",
             "source": "https://archive.org/details/vrajeshkumar_gmail_061/01-%E0%AE%AA%E0%AE%BE%E0%AE%B2%20%E0%AE%95%E0%AE%BE%E0%AE%A3%E0%AF%8D%E0%AE%9F%E0%AE%AE%E0%AF%8D/page/n15/mode/2up",
@@ -249,3 +292,21 @@ class SanatanConfig:
             ],
         },
     ]
+
+    def get_scripture_by_collection(self, collection_name: str):
+        return [
+            scripture
+            for scripture in self.scriptures
+            if scripture["collection_name"] == collection_name
+        ][0]
+
+    def is_metadata_field_allowed(self, collection_name: str, metadata_field: str):
+        scripture = self.get_scripture_by_collection(collection_name=collection_name)
+        if metadata_field not in [
+            field["name"] for field in scripture["metadata_fields"]
+        ]:
+            raise Exception(
+                f"metadata_field: [{metadata_field}] not allowed in collection [{collection_name}]. Here are the allowed fields with their descriptions: {scripture["metadata_fields"]}"
+            )
+        else:
+            return True

graph_helper.py

@@ -6,6 +6,7 @@ from langchain_openai import ChatOpenAI
 from langgraph.graph.state import CompiledStateGraph
 from tools import (
     tool_format_scripture_answer,
+    tool_get_standardized_prabandham_names,
     tool_search_db,
     tool_search_web,
     tool_push,
@@ -33,6 +34,7 @@ def generate_graph() -> CompiledStateGraph:
         tool_search_db,
         tool_format_scripture_answer,
         tool_get_standardized_azhwar_names,
+        tool_get_standardized_prabandham_names,
         tool_get_standardized_divya_desam_names,
         tool_search_db_by_metadata,
     ]

sanatan_assistant.py

@@ -15,7 +15,7 @@ allowedCollections = Literal[
 
 
 def format_scripture_answer(
-    scripture_title: allowedCollections, question: str, query_tool_output: str
+    collection_name: allowedCollections, question: str, query_tool_output: str
 ):
     """
     Use this tool to generate a custom system prompt based on the scripture title, question, and query_tool_output.
@@ -25,14 +25,14 @@ def format_scripture_answer(
     The generated prompt will guide the assistant to respond using only that scripture’s content, with a clear format including Sanskrit/Tamil verses, English explanations, and source chapters.
     """
 
-    prompt = f"""You are a knowledgeable assistant on the scripture *{scripture_title}*, well-versed in **Sanskrit** and **English** and **Tamil**.
+    prompt = f"""You are a knowledgeable assistant on the scripture *{collection_name}*, well-versed in **Sanskrit** , **English** and **Tamil**.
 
-You must answer the question using **only** the content from *{scripture_title}* provided in the context below.  
+You must answer the question using **only** the content from *{collection_name}* provided in the context below.  
 - Do **not** bring in information from **any other scripture or source**, or from prior knowledge, even if the answer seems obvious or well-known.  
 - Do **not** quote any Sanskrit/Tamil verses unless they appear **explicitly** in the provided context.  
 - Do **not** use verse numbers or line references unless clearly mentioned in the context.  
 - If the answer cannot be found in the context, clearly say:  
-  **"I do not have enough information from the {scripture_title} to answer this."**
+  **"I do not have enough information from the {collection_name} to answer this."**
 
 If the answer is not directly stated in the verses but is present in explanatory notes within the context, you may interpret — but **explicitly mention that it is an interpretation**.
 
@@ -43,18 +43,27 @@ If the user query is not small talk, use the following response format (in Markd
 - If it is an interpretation, say so explicitly.
 
 ### 🔍 Scripture  
-- Mention the scripture from which the references were taken.
+- Mention the scripture(s) from which the references were taken.
 
 ### 🔍 Chapter  
 - Mention the chapter(s) from which the references were taken.  
 - Only mention chapters if they are explicitly part of the context.
 
+### 🔍 Verse Number
+- Mention the *verse number* from which the references were taken.  
+
+### 🔍 Reference Link(s)
+- Provide reference link(s) (`html_url`) if one is available in the context.
+
 ### 📜 Supporting Sanskrit/Tamil Verse(s)  
-- Quote **only the most relevant** Sanskrit/Tamil verse(s) from the context.  
+- Quote the **original** native verse(s) from the context as they were without any manipulation or transliteration.
 - Do **not** add verses from memory or outside the context.  
-- Format each verse clearly, one per line.  
-- **Avoid transliteration unless no Devanagari/Tamil is available.**  
-- Do **not** provide English text in this section.
+- Format each verse clearly.  
+- Do **not** provide English translation or transliterated text in this section.
+
+### 🔍 English Transliteration
+- Provide the **corresponding English transliteration** for each Sanskrit/Tamil verse shown.  
+- Keep it readable and aligned with the verse above.
 
 ### 🔍 English Translation  
 - Provide the **corresponding English meaning** for each Sanskrit/Tamil verse shown.  
@@ -163,7 +172,7 @@ def query_by_metadata_field(
     - collection_name (str): The name of the scripture collection to search. ...
     - query (str): The search query.
     - metadata_field (str) : The name of the metadata field. e.g. azhwar_name
-    - metadata_search_operator (str) : The search operator e.g. $eq or $in
+    - metadata_search_operator (str) : The search operator e.g. $eq or $in. DO NOT use $regex.
     - metadata_value : Value to search for can be any primitive datatype like str or int (or a list[str] if metadata_search_operator = '$in'). for e.g. Thirumangai Azhwar or '2233' or 2233
     - n_results (int): Number of results to return. Default is 5.
 
@@ -171,6 +180,12 @@ def query_by_metadata_field(
     - A list of matching results.
     """
     logger.info("Searching collection [%s] for [%s]", collection_name, query)
+
+    try:
+        SanatanConfig().is_metadata_field_allowed(collection_name=collection_name, metadata_field=metadata_field)
+    except:
+        raise
+
     response = sanatanDatabase.search_by_metadata(
         collection_name=collection_name,
         query=query,

tools.py

@@ -20,8 +20,8 @@ tool_search_db = StructuredTool.from_function(
         "Do a vector Search within a specific scripture collection. "
         f"The collection_name must be one of: {', '.join(allowed_collections)}."
         "Use this to find relevant scripture verses or explanations based on the given query."
-        "If the query doesn't yield any relevant results, then call `tool_search_db_by_metadata` tool to search specifically by a given metadata field (only if specific field from metadata has been mentioned)."
-        f"use this configuration for reference :\n{json.dumps(SanatanConfig.scriptures, indent=1)}\n"
+        # "If the query doesn't yield any relevant results, then call `tool_search_db_by_metadata` tool to search specifically by a given metadata field (only if specific field from metadata has been mentioned)."
+        # f"use this configuration for reference :\n{json.dumps(SanatanConfig.scriptures, indent=1)}\n"
     ),
 )
 
@@ -59,6 +59,16 @@ tool_get_standardized_azhwar_names = StructuredTool.from_function(
     ),
 )
 
+tool_get_standardized_prabandham_names = StructuredTool.from_function(
+    get_standardized_azhwar_names,
+    name="tool_get_standardized_prabandham_names",
+    description=(
+        "Get a list of standardized  prabandham names."
+        "Use this tool to standardize the names of the prabandhams when the user asks for pasurams from a specific prabandham."
+        "Usually this is followed by passing that `prabandham_name` for a metadata search using the `tool_search_db_by_metadata` tool."
+        "if you are passing the prabandham_code, make sure you use metadata field `prabandham` and NOT `prabandham_code` as argument to `tool_search_db_by_metadata` tool."
+    ),
+)
 
 tool_get_standardized_divya_desam_names = StructuredTool.from_function(
     get_standardized_divya_desam_names,