Update scripts/setup_vectorstore.py

scripts/setup_vectorstore.py

@@ -1,30 +1,21 @@
-from langchain_community.vectorstores import Chroma
-from langchain_openai import OpenAIEmbeddings
-import pickle
-import os
-
-CHUNKS_PATH = "E:/courses/LangChain Project/main root/output/chunks.pkl"
-DB_DIR = "E:/courses/LangChain Project/main root/db"
-BATCH_SIZE = 100  # You can tune this depending on average token size per chunk
-
-if not os.path.exists(CHUNKS_PATH):
-    raise FileNotFoundError("Run chunk_and_embed.py first")
-
-with open(CHUNKS_PATH, "rb") as f:
-    chunks = pickle.load(f)
-
-embedding = OpenAIEmbeddings(model="text-embedding-3-small")
-
-# Create or load the vectorstore
-vectorstore = Chroma(persist_directory=DB_DIR, embedding_function=embedding)
-
-print(f"🧠 Embedding and adding {len(chunks)} chunks in batches of {BATCH_SIZE}...")
-
-# Add documents in batches to avoid hitting token limits
-for i in range(0, len(chunks), BATCH_SIZE):
-    batch = chunks[i:i + BATCH_SIZE]
-    vectorstore.add_documents(batch)
-    print(f"✅ Added batch {i // BATCH_SIZE + 1} of {len(chunks) // BATCH_SIZE + 1}")
-
-# vectorstore.persist()
-print(f"✅ Vectorstore saved to {DB_DIR}")
+import pickle
+from pathlib import Path
+from langchain_community.vectorstores import Chroma
+from langchain_openai import OpenAIEmbeddings
+
+BASE_DIR = Path(__file__).resolve().parent.parent
+CHUNKS_PATH = BASE_DIR / "output" / "chunks.pkl"
+DB_DIR = BASE_DIR / "db"
+
+with open(CHUNKS_PATH, "rb") as f:
+    chunks = pickle.load(f)
+
+embedding = OpenAIEmbeddings(model="text-embedding-3-small")
+vectorstore = Chroma(persist_directory=str(DB_DIR), embedding_function=embedding)
+
+BATCH_SIZE = 100
+print(f"🧠 Embedding and adding {len(chunks)} chunks in batches...")
+for i in range(0, len(chunks), BATCH_SIZE):
+    batch = chunks[i:i + BATCH_SIZE]
+    vectorstore.add_documents(batch)
+    print(f"✅ Added batch {i // BATCH_SIZE + 1}")