Add vector database creation configuration and update related scripts

.env.example

@@ -27,3 +27,9 @@ MAX_SEARCH_RESULTS=5
 # Document Chunking Configuration
 CHUNK_SIZE=1000
 CHUNK_OVERLAP=200
+
+# Vector Database Creation Configuration
+FORCE_RECREATE=False
+OUTPUT_DIR=./stats
+USE_CHUNKING=True
+SHOULD_SAVE_STATS=True

CONTRIBUTING.md

@@ -29,6 +29,12 @@ TheDataGuy Chat is a Q&A chatbot powered by the content from [TheDataGuy blog](h
    VECTOR_STORAGE_PATH=./db/vector_store_tdg
    LLM_MODEL=gpt-4o-mini
    EMBEDDING_MODEL=Snowflake/snowflake-arctic-embed-l
+   
+   # Vector Database Creation Configuration (optional)
+   FORCE_RECREATE=False      # Whether to force recreation of the vector store
+   OUTPUT_DIR=./stats        # Directory to save stats and artifacts
+   USE_CHUNKING=True         # Whether to split documents into chunks
+   SHOULD_SAVE_STATS=True    # Whether to save statistics about the documents
    ```
 
 3. Install dependencies:

README.md

@@ -94,6 +94,16 @@ CHUNK_SIZE=1000
 CHUNK_OVERLAP=200
 ```
 
+Additional configuration options for vector database creation:
+
+```
+# Vector Database Creation Configuration
+FORCE_RECREATE=False      # Whether to force recreation of the vector store
+OUTPUT_DIR=./stats        # Directory to save stats and artifacts
+USE_CHUNKING=True         # Whether to split documents into chunks
+SHOULD_SAVE_STATS=True    # Whether to save statistics about the documents
+```
+
 ## Running Locally
 
 ### Using Docker

py-src/app.py

@@ -10,7 +10,8 @@ load_dotenv()
 import pipeline
 #build vector store
 print("=== create vector db ===")
-pipeline.create_vector_database(force_recreate=True,save_stats=False,use_chunking=True)
+# Use configuration from config rather than hardcoded values
+pipeline.create_vector_database()
 print("========================")
 
 import chainlit as cl

py-src/lets_talk/config.py

@@ -20,5 +20,11 @@ MAX_SEARCH_RESULTS = int(os.environ.get("MAX_SEARCH_RESULTS", "5"))
 CHUNK_SIZE = int(os.environ.get("CHUNK_SIZE", "1000"))
 CHUNK_OVERLAP = int(os.environ.get("CHUNK_OVERLAP", "200"))
 
+# Vector database creation configuration
+FORCE_RECREATE = os.environ.get("FORCE_RECREATE", "False").lower() == "true"
+OUTPUT_DIR = os.environ.get("OUTPUT_DIR", "./stats")
+USE_CHUNKING = os.environ.get("USE_CHUNKING", "True").lower() == "true"
+SHOULD_SAVE_STATS = os.environ.get("SHOULD_SAVE_STATS", "True").lower() == "true"
+
 
 

py-src/pipeline.py

@@ -21,7 +21,10 @@ from datetime import datetime
 import json
 import logging
 from pathlib import Path
-from lets_talk.config import VECTOR_STORAGE_PATH, DATA_DIR
+from lets_talk.config import (
+    CHUNK_OVERLAP, CHUNK_SIZE, VECTOR_STORAGE_PATH, DATA_DIR,
+    FORCE_RECREATE, OUTPUT_DIR, USE_CHUNKING, SHOULD_SAVE_STATS
+)
 
 # Import the blog utilities module
 import lets_talk.utils.blog as blog
@@ -100,19 +103,20 @@ def save_stats(stats, output_dir="./stats", ci_mode=False):
     return filename, basic_stats
 
 def create_vector_database(data_dir=DATA_DIR, storage_path=VECTOR_STORAGE_PATH, 
-                      force_recreate=False, output_dir="./stats", ci_mode=False, 
-                      use_chunking=True, should_save_stats=True, chunk_size=None, chunk_overlap=None):
+                      force_recreate=FORCE_RECREATE, output_dir=OUTPUT_DIR, ci_mode=False, 
+                      use_chunking=USE_CHUNKING, should_save_stats=SHOULD_SAVE_STATS, 
+                      chunk_size=CHUNK_SIZE, chunk_overlap=CHUNK_OVERLAP):
     """
     Create or update the vector database with blog documents.
     
     Args:
-        data_dir: Directory containing the blog posts
-        storage_path: Path where the vector database will be stored
-        force_recreate: Whether to force recreation of the vector store
-        output_dir: Directory to save stats and artifacts
+        data_dir: Directory containing the blog posts (default from config)
+        storage_path: Path where the vector database will be stored (default from config)
+        force_recreate: Whether to force recreation of the vector store (default from config)
+        output_dir: Directory to save stats and artifacts (default from config)
         ci_mode: Whether to run in CI mode
-        use_chunking: Whether to split documents into chunks
-        should_save_stats: Whether to save statistics about the documents
+        use_chunking: Whether to split documents into chunks (default from config)
+        should_save_stats: Whether to save statistics about the documents (default from config)
         chunk_size: Size of each chunk in characters (default from config)
         chunk_overlap: Overlap between chunks in characters (default from config)
         

scripts/build-vector-store.sh

@@ -1,14 +1,21 @@
 #!/bin/bash
 # Script to build vector store locally
 # Usage: ./scripts/build-vector-store.sh [--force-recreate]
+#
+# Environment variables that can be set:
+#   FORCE_RECREATE   - Set to "true" to force recreation of the vector store
+#   OUTPUT_DIR       - Directory to save stats and artifacts (default: ./artifacts)
+#   USE_CHUNKING     - Set to "false" to disable document chunking
+#   SHOULD_SAVE_STATS - Set to "false" to disable saving document statistics
 
+# Parse command line arguments
 FORCE_RECREATE=""
 if [[ "$1" == "--force-recreate" ]]; then
   FORCE_RECREATE="--force-recreate"
 fi
 
-# Set output directory for artifacts
-OUTPUT_DIR="./artifacts"
+# Set output directory for artifacts (use environment variable if set)
+OUTPUT_DIR=${OUTPUT_DIR:-"./artifacts"}
 mkdir -p $OUTPUT_DIR
 
 echo "Building vector store with output to $OUTPUT_DIR"