Upload folder using huggingface_hub

config.py

@@ -16,7 +16,8 @@ class SanatanConfig:
             "name": "vishnu_puranam",
             "title": "Sri Vishnu Puranam",
             "output_dir": "./output/vishnu_puranam",
-            "collection_name": "vishnu_puranam",
+            "collection_name": "vishnu_puranam_openai",
+            "collection_embedding_fn": "openai",
             "metadata_fields": [
                 {"name": "file", "datatype": "str"},
                 {"name": "num_chars", "datatype": "str"},

db.py

@@ -7,6 +7,7 @@ from pydantic import BaseModel
 
 from metadata import MetadataFilter, MetadataWhereClause
 from modules.db.relevance import validate_relevance_queryresult
+from tqdm import tqdm
 
 logger = logging.getLogger(__name__)
 logger.setLevel(logging.INFO)
@@ -53,7 +54,7 @@ class SanatanDatabase:
             )
    
         validated_response = validate_relevance_queryresult(query, response)
-        
+
         return validated_response["result"]
 
     def search_for_literal(
@@ -175,6 +176,54 @@ class SanatanDatabase:
             count = self.count(collection_name=scripture["collection_name"])
             if count == 0:
                 raise Exception(f"No data in collection {scripture["collection_name"]}")
+    
+    def reembed_collection_openai(self, collection_name: str, batch_size: int = 50):
+        """
+        Deletes and recreates a Chroma collection with OpenAI text-embedding-3-large embeddings.
+        All existing documents are re-embedded and inserted into the new collection.
+        
+        Args:
+            collection_name: The name of the collection to delete/recreate.
+            batch_size: Number of documents to process per batch.
+        """
+        # Step 1: Fetch old collection data (if exists)
+        try:
+            old_collection = self.chroma_client.get_collection(name=collection_name)
+            old_data = old_collection.get(include=["documents", "metadatas"])
+            documents = old_data["documents"]
+            metadatas = old_data["metadatas"]
+            ids = old_data["ids"]
+            print(f"Fetched {len(documents)} documents from old collection.")
+            
+            # Step 2: Delete old collection
+            # self.chroma_client.delete_collection(collection_name)
+            # print(f"Deleted old collection '{collection_name}'.")
+        except chromadb.errors.NotFoundError:
+            print(f"No existing collection named '{collection_name}', starting fresh.")
+            documents, metadatas, ids = [], [], []
+
+        # Step 3: Create new collection with correct embedding dimension
+        new_collection = self.chroma_client.create_collection(
+            name=f"{collection_name}_openai",
+            embedding_function=None,  # embeddings will be provided manually
+        )
+        print(f"Created new collection '{collection_name}_openai' with embedding_dim=3072.")
+
+        # Step 4: Re-embed and insert documents in batches
+        for i in tqdm(range(0, len(documents), batch_size), desc="Re-embedding batches"):
+            batch_docs = documents[i:i+batch_size]
+            batch_metadatas = metadatas[i:i+batch_size]
+            batch_ids = ids[i:i+batch_size]
+
+            embeddings = get_embedding(batch_docs, backend="openai")
+
+            new_collection.add(
+                ids=batch_ids,
+                documents=batch_docs,
+                metadatas=batch_metadatas,
+                embeddings=embeddings
+            )
+        print("All documents re-embedded and added to new collection successfully!")
 
 
 if __name__ == "__main__":
@@ -213,4 +262,4 @@ if __name__ == "__main__":
         # print("Document: ")
         # print(response["documents"][0][0])
         # print("Metadata: ")
-        # print(response["metadatas"][0][0])
+        # print(response["metadatas"][0][0])

modules/db/reembed.py

@@ -0,0 +1,4 @@
+from db import SanatanDatabase
+
+if __name__ == "__main__":
+    SanatanDatabase().reembed_collection_openai(collection_name="vishnu_puranam")