Few new models

.gitignore

@@ -1 +1,2 @@
 .env
+/__pycache__

load_existing_docs.py

@@ -6,12 +6,7 @@ from langchain_community.document_loaders import PyPDFLoader, UnstructuredURLLoa
 from qdrant_client.http.models import VectorParams
 import pymupdf
 import requests
-
-#qdrant = QdrantVectorStore.from_existing_collection(
-#    embedding=models.basic_embeddings,
-#    collection_name="kai_test_documents",
-#    url=constants.QDRANT_ENDPOINT,
-#)
+from transformers import AutoTokenizer
 
 def extract_links_from_pdf(pdf_path):
     links = []
@@ -78,26 +73,22 @@ for link in unique_links:
 
 
 #print(len(documents))
-semantic_split_docs = models.semanticChunker.split_documents(documents)
-RCTS_split_docs = models.RCTS.split_documents(documents)
-
-
-#for file in filepaths:
-#    loader = PyPDFLoader(file)
-#    documents = loader.load()
-#    for doc in documents:
-#        doc.metadata = {
-#            "source": file,
-#            "tag": "employee" if "employee" in file.lower() else "employer"
-#        }
-#    all_documents.extend(documents) 
-
-#chunk them
-#semantic_split_docs = models.semanticChunker.split_documents(all_documents)
-
-
+#semantic_split_docs = models.semanticChunker.split_documents(documents)
+semantic_tuned_split_docs = models.semanticChunker_tuned.split_documents(documents)
+#RCTS_split_docs = models.RCTS.split_documents(documents)
+#print(len(semantic_split_docs))
+print(len(semantic_tuned_split_docs))
+#tokenizer = models.tuned_embeddings.client.tokenizer
+#
+#token_sizes = [len(tokenizer.encode(chunk)) for chunk in semantic_tuned_split_docs]
+
+# Display the token sizes
+#for idx, size in enumerate(token_sizes):
+#    print(f"Chunk {idx + 1}: {size} tokens")
+#
+#exit()
 #add them to the existing qdrant client
-collection_name = "docs_from_ripped_urls_recursive"
+collection_name = "docs_from_ripped_urls_semantic_tuned"
 
 collections = models.qdrant_client.get_collections()
 collection_names = [collection.name for collection in collections.collections]
@@ -105,16 +96,16 @@ collection_names = [collection.name for collection in collections.collections]
 if collection_name not in collection_names:
     models.qdrant_client.create_collection(
         collection_name=collection_name,
-        vectors_config=VectorParams(size=1536, distance="Cosine") 
+        vectors_config=VectorParams(size=1024, distance="Cosine") 
     )
 
 qdrant_vector_store = QdrantVectorStore(
     client=models.qdrant_client,
     collection_name=collection_name,
-    embedding=models.te3_small
+    embedding=models.tuned_embeddings
 )
 
-qdrant_vector_store.add_documents(RCTS_split_docs)
+qdrant_vector_store.add_documents(semantic_tuned_split_docs)
 
 
 

models.py

@@ -5,9 +5,11 @@ from langchain.callbacks.tracers import LangChainTracer
 from langchain_huggingface.embeddings import HuggingFaceEmbeddings
 from langchain_experimental.text_splitter import SemanticChunker
 from langchain_openai.embeddings import OpenAIEmbeddings
-from langchain_community.vectorstores import Qdrant
+from langchain_qdrant import QdrantVectorStore, Qdrant
+from langchain.retrievers.contextual_compression import ContextualCompressionRetriever
 from qdrant_client import QdrantClient
 from langchain_text_splitters import RecursiveCharacterTextSplitter
+from langchain_cohere import CohereRerank
 import constants
 import os
 
@@ -66,10 +68,8 @@ gpt4o_mini = ChatOpenAI(
 )
 
 basic_embeddings = HuggingFaceEmbeddings(model_name="snowflake/snowflake-arctic-embed-l")
-#hkunlp_instructor_large = HuggingFaceInstructEmbeddings(
-#    model_name = "hkunlp/instructor-large",
-#    query_instruction="Represent the query for retrieval: "
-#)
+
+tuned_embeddings = HuggingFaceEmbeddings(model_name="CoExperiences/snowflake-l-marketing-tuned")
 
 te3_small = OpenAIEmbeddings(api_key=constants.OPENAI_API_KEY, model="text-embedding-3-small")
 
@@ -78,9 +78,28 @@ semanticChunker = SemanticChunker(
     breakpoint_threshold_type="percentile"
 )
 
+semanticChunker_tuned = SemanticChunker(
+    tuned_embeddings,
+    breakpoint_threshold_type="percentile",
+    breakpoint_threshold_amount=85
+)
+
 RCTS = RecursiveCharacterTextSplitter(
     # Set a really small chunk size, just to show.
     chunk_size=500,
     chunk_overlap=25,
     length_function=len,
+)
+
+semantic_tuned_Qdrant_vs = QdrantVectorStore(
+    client=qdrant_client,
+    collection_name="docs_from_ripped_urls_semantic_tuned",
+    embedding=tuned_embeddings
+)
+semantic_tuned_retriever = semantic_tuned_Qdrant_vs.as_retriever(search_kwargs={"k" : 10})
+
+#compression
+compressor = CohereRerank(model="rerank-english-v3.0")
+compression_retriever = ContextualCompressionRetriever(
+    base_compressor=compressor, base_retriever=semantic_tuned_retriever
 )

tuning/requirements.in

@@ -0,0 +1,14 @@
+langchain_openai
+langchain_huggingface
+langchain_core==0.2.38
+langchain
+langchain_community
+langchain-text-splitters
+faiss-cpu
+unstructured==0.15.7
+python-pptx==1.0.2
+nltk==3.9.1
+pyarrow
+sentence_transformers
+datasets
+ragas

tuning/requirements.txt

@@ -0,0 +1,412 @@
+#
+# This file is autogenerated by pip-compile with Python 3.11
+# by the following command:
+#
+#    pip-compile requirements.in
+#
+aiohappyeyeballs==2.4.3
+    # via aiohttp
+aiohttp==3.10.10
+    # via
+    #   datasets
+    #   fsspec
+    #   langchain
+    #   langchain-community
+aiosignal==1.3.1
+    # via aiohttp
+annotated-types==0.7.0
+    # via pydantic
+anyio==4.6.2.post1
+    # via
+    #   httpx
+    #   openai
+appdirs==1.4.4
+    # via ragas
+attrs==24.2.0
+    # via aiohttp
+backoff==2.2.1
+    # via unstructured
+beautifulsoup4==4.12.3
+    # via unstructured
+certifi==2024.8.30
+    # via
+    #   httpcore
+    #   httpx
+    #   requests
+cffi==1.17.1
+    # via cryptography
+chardet==5.2.0
+    # via unstructured
+charset-normalizer==3.4.0
+    # via requests
+click==8.1.7
+    # via nltk
+cryptography==43.0.1
+    # via unstructured-client
+dataclasses-json==0.6.7
+    # via
+    #   langchain-community
+    #   unstructured
+datasets==3.0.1
+    # via
+    #   -r requirements.in
+    #   ragas
+dill==0.3.8
+    # via
+    #   datasets
+    #   multiprocess
+distro==1.9.0
+    # via openai
+emoji==2.14.0
+    # via unstructured
+eval-type-backport==0.2.0
+    # via unstructured-client
+faiss-cpu==1.9.0
+    # via -r requirements.in
+filelock==3.16.1
+    # via
+    #   datasets
+    #   huggingface-hub
+    #   torch
+    #   transformers
+    #   triton
+filetype==1.2.0
+    # via unstructured
+frozenlist==1.4.1
+    # via
+    #   aiohttp
+    #   aiosignal
+fsspec[http]==2024.6.1
+    # via
+    #   datasets
+    #   huggingface-hub
+    #   torch
+greenlet==3.1.1
+    # via sqlalchemy
+h11==0.14.0
+    # via httpcore
+httpcore==1.0.6
+    # via httpx
+httpx==0.27.2
+    # via
+    #   langsmith
+    #   openai
+    #   unstructured-client
+huggingface-hub==0.26.0
+    # via
+    #   datasets
+    #   langchain-huggingface
+    #   sentence-transformers
+    #   tokenizers
+    #   transformers
+idna==3.10
+    # via
+    #   anyio
+    #   httpx
+    #   requests
+    #   yarl
+jinja2==3.1.4
+    # via torch
+jiter==0.6.1
+    # via openai
+joblib==1.4.2
+    # via
+    #   nltk
+    #   scikit-learn
+jsonpatch==1.33
+    # via langchain-core
+jsonpath-python==1.0.6
+    # via unstructured-client
+jsonpointer==3.0.0
+    # via jsonpatch
+langchain==0.2.16
+    # via
+    #   -r requirements.in
+    #   langchain-community
+    #   ragas
+langchain-community==0.2.16
+    # via
+    #   -r requirements.in
+    #   ragas
+langchain-core==0.2.38
+    # via
+    #   -r requirements.in
+    #   langchain
+    #   langchain-community
+    #   langchain-huggingface
+    #   langchain-openai
+    #   langchain-text-splitters
+    #   ragas
+langchain-huggingface==0.0.3
+    # via -r requirements.in
+langchain-openai==0.1.23
+    # via
+    #   -r requirements.in
+    #   ragas
+langchain-text-splitters==0.2.4
+    # via
+    #   -r requirements.in
+    #   langchain
+langdetect==1.0.9
+    # via unstructured
+langsmith==0.1.136
+    # via
+    #   langchain
+    #   langchain-community
+    #   langchain-core
+lxml==5.3.0
+    # via
+    #   python-pptx
+    #   unstructured
+markupsafe==3.0.2
+    # via jinja2
+marshmallow==3.23.0
+    # via dataclasses-json
+mpmath==1.3.0
+    # via sympy
+multidict==6.1.0
+    # via
+    #   aiohttp
+    #   yarl
+multiprocess==0.70.16
+    # via datasets
+mypy-extensions==1.0.0
+    # via typing-inspect
+nest-asyncio==1.6.0
+    # via
+    #   ragas
+    #   unstructured-client
+networkx==3.4.1
+    # via torch
+nltk==3.9.1
+    # via
+    #   -r requirements.in
+    #   unstructured
+numpy==1.26.4
+    # via
+    #   datasets
+    #   faiss-cpu
+    #   langchain
+    #   langchain-community
+    #   pandas
+    #   pyarrow
+    #   ragas
+    #   scikit-learn
+    #   scipy
+    #   transformers
+    #   unstructured
+nvidia-cublas-cu12==12.4.5.8
+    # via
+    #   nvidia-cudnn-cu12
+    #   nvidia-cusolver-cu12
+    #   torch
+nvidia-cuda-cupti-cu12==12.4.127
+    # via torch
+nvidia-cuda-nvrtc-cu12==12.4.127
+    # via torch
+nvidia-cuda-runtime-cu12==12.4.127
+    # via torch
+nvidia-cudnn-cu12==9.1.0.70
+    # via torch
+nvidia-cufft-cu12==11.2.1.3
+    # via torch
+nvidia-curand-cu12==10.3.5.147
+    # via torch
+nvidia-cusolver-cu12==11.6.1.9
+    # via torch
+nvidia-cusparse-cu12==12.3.1.170
+    # via
+    #   nvidia-cusolver-cu12
+    #   torch
+nvidia-nccl-cu12==2.21.5
+    # via torch
+nvidia-nvjitlink-cu12==12.4.127
+    # via
+    #   nvidia-cusolver-cu12
+    #   nvidia-cusparse-cu12
+    #   torch
+nvidia-nvtx-cu12==12.4.127
+    # via torch
+openai==1.52.0
+    # via
+    #   langchain-openai
+    #   ragas
+orjson==3.10.7
+    # via langsmith
+packaging==24.1
+    # via
+    #   datasets
+    #   faiss-cpu
+    #   huggingface-hub
+    #   langchain-core
+    #   marshmallow
+    #   transformers
+pandas==2.2.3
+    # via datasets
+pillow==11.0.0
+    # via
+    #   python-pptx
+    #   sentence-transformers
+propcache==0.2.0
+    # via yarl
+psutil==6.1.0
+    # via unstructured
+pyarrow==17.0.0
+    # via
+    #   -r requirements.in
+    #   datasets
+pycparser==2.22
+    # via cffi
+pydantic==2.9.2
+    # via
+    #   langchain
+    #   langchain-core
+    #   langsmith
+    #   openai
+    #   ragas
+    #   unstructured-client
+pydantic-core==2.23.4
+    # via pydantic
+pypdf==5.0.1
+    # via unstructured-client
+pysbd==0.3.4
+    # via ragas
+python-dateutil==2.8.2
+    # via
+    #   pandas
+    #   unstructured-client
+python-iso639==2024.4.27
+    # via unstructured
+python-magic==0.4.27
+    # via unstructured
+python-pptx==1.0.2
+    # via -r requirements.in
+pytz==2024.2
+    # via pandas
+pyyaml==6.0.2
+    # via
+    #   datasets
+    #   huggingface-hub
+    #   langchain
+    #   langchain-community
+    #   langchain-core
+    #   transformers
+ragas==0.2.1
+    # via -r requirements.in
+rapidfuzz==3.10.0
+    # via unstructured
+regex==2024.9.11
+    # via
+    #   nltk
+    #   tiktoken
+    #   transformers
+requests==2.32.3
+    # via
+    #   datasets
+    #   huggingface-hub
+    #   langchain
+    #   langchain-community
+    #   langsmith
+    #   requests-toolbelt
+    #   tiktoken
+    #   transformers
+    #   unstructured
+requests-toolbelt==1.0.0
+    # via
+    #   langsmith
+    #   unstructured-client
+safetensors==0.4.5
+    # via transformers
+scikit-learn==1.5.2
+    # via sentence-transformers
+scipy==1.14.1
+    # via
+    #   scikit-learn
+    #   sentence-transformers
+sentence-transformers==3.2.0
+    # via
+    #   -r requirements.in
+    #   langchain-huggingface
+six==1.16.0
+    # via
+    #   langdetect
+    #   python-dateutil
+sniffio==1.3.1
+    # via
+    #   anyio
+    #   httpx
+    #   openai
+soupsieve==2.6
+    # via beautifulsoup4
+sqlalchemy==2.0.36
+    # via
+    #   langchain
+    #   langchain-community
+sympy==1.13.1
+    # via torch
+tabulate==0.9.0
+    # via unstructured
+tenacity==8.5.0
+    # via
+    #   langchain
+    #   langchain-community
+    #   langchain-core
+threadpoolctl==3.5.0
+    # via scikit-learn
+tiktoken==0.8.0
+    # via
+    #   langchain-openai
+    #   ragas
+tokenizers==0.20.1
+    # via
+    #   langchain-huggingface
+    #   transformers
+torch==2.5.0
+    # via sentence-transformers
+tqdm==4.66.5
+    # via
+    #   datasets
+    #   huggingface-hub
+    #   nltk
+    #   openai
+    #   sentence-transformers
+    #   transformers
+    #   unstructured
+transformers==4.45.2
+    # via
+    #   langchain-huggingface
+    #   sentence-transformers
+triton==3.1.0
+    # via torch
+typing-extensions==4.12.2
+    # via
+    #   huggingface-hub
+    #   langchain-core
+    #   openai
+    #   pydantic
+    #   pydantic-core
+    #   python-pptx
+    #   sqlalchemy
+    #   torch
+    #   typing-inspect
+    #   unstructured
+typing-inspect==0.9.0
+    # via
+    #   dataclasses-json
+    #   unstructured-client
+tzdata==2024.2
+    # via pandas
+unstructured==0.15.7
+    # via -r requirements.in
+unstructured-client==0.26.1
+    # via unstructured
+urllib3==2.2.3
+    # via requests
+wrapt==1.16.0
+    # via unstructured
+xlsxwriter==3.2.0
+    # via python-pptx
+xxhash==3.5.0
+    # via datasets
+yarl==1.15.4
+    # via aiohttp