scripts/chunk_and_embed.py

from langchain.text_splitter import RecursiveCharacterTextSplitter
from pathlib import Path
import pickle
import os


BASE_DIR = Path(__file__).resolve().parent.parent
DOCS_PATH = BASE_DIR / "output" / "all_docs.pkl"
CHUNKS_PATH = BASE_DIR / "output" / "chunks.pkl"

with open(DOCS_PATH, "rb") as f:
    all_docs = pickle.load(f)

splitter = RecursiveCharacterTextSplitter(chunk_size=1000, chunk_overlap=100)
chunks = splitter.split_documents(all_docs)
print(f"✅ Split into {len(chunks)} chunks")

with open(CHUNKS_PATH, "wb") as f:
    pickle.dump(chunks, f)
print(f"📦 Chunks saved to {CHUNKS_PATH}")