Spaces:
Sleeping
Sleeping
File size: 606 Bytes
1458fa6 18a6316 1458fa6 5f096cc |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 |
from langchain.text_splitter import RecursiveCharacterTextSplitter
from pathlib import Path
import pickle
import os
BASE_DIR = Path(__file__).resolve().parent.parent
DOCS_PATH = BASE_DIR / "output" / "all_docs.pkl"
CHUNKS_PATH = BASE_DIR / "output" / "chunks.pkl"
with open(DOCS_PATH, "rb") as f:
all_docs = pickle.load(f)
splitter = RecursiveCharacterTextSplitter(chunk_size=1000, chunk_overlap=100)
chunks = splitter.split_documents(all_docs)
print(f"✅ Split into {len(chunks)} chunks")
with open(CHUNKS_PATH, "wb") as f:
pickle.dump(chunks, f)
print(f"📦 Chunks saved to {CHUNKS_PATH}") |