Merge branch 'main' of https://huggingface.co/spaces/CoExperiences/aie4-final

.gitignore

@@ -1 +1,2 @@
 .env
+/__pycache__

load_existing_docs.py

@@ -6,12 +6,7 @@ from langchain_community.document_loaders import PyPDFLoader, UnstructuredURLLoa
 from qdrant_client.http.models import VectorParams
 import pymupdf
 import requests
-
-#qdrant = QdrantVectorStore.from_existing_collection(
-#    embedding=models.basic_embeddings,
-#    collection_name="kai_test_documents",
-#    url=constants.QDRANT_ENDPOINT,
-#)
+from transformers import AutoTokenizer
 
 def extract_links_from_pdf(pdf_path):
     links = []
@@ -78,26 +73,22 @@ for link in unique_links:
 
 
 #print(len(documents))
-semantic_split_docs = models.semanticChunker.split_documents(documents)
-RCTS_split_docs = models.RCTS.split_documents(documents)
-
-
-#for file in filepaths:
-#    loader = PyPDFLoader(file)
-#    documents = loader.load()
-#    for doc in documents:
-#        doc.metadata = {
-#            "source": file,
-#            "tag": "employee" if "employee" in file.lower() else "employer"
-#        }
-#    all_documents.extend(documents) 
-
-#chunk them
-#semantic_split_docs = models.semanticChunker.split_documents(all_documents)
-
-
+#semantic_split_docs = models.semanticChunker.split_documents(documents)
+semantic_tuned_split_docs = models.semanticChunker_tuned.split_documents(documents)
+#RCTS_split_docs = models.RCTS.split_documents(documents)
+#print(len(semantic_split_docs))
+print(len(semantic_tuned_split_docs))
+#tokenizer = models.tuned_embeddings.client.tokenizer
+#
+#token_sizes = [len(tokenizer.encode(chunk)) for chunk in semantic_tuned_split_docs]
+
+# Display the token sizes
+#for idx, size in enumerate(token_sizes):
+#    print(f"Chunk {idx + 1}: {size} tokens")
+#
+#exit()
 #add them to the existing qdrant client
-collection_name = "docs_from_ripped_urls_recursive"
+collection_name = "docs_from_ripped_urls_semantic_tuned"
 
 collections = models.qdrant_client.get_collections()
 collection_names = [collection.name for collection in collections.collections]
@@ -105,16 +96,16 @@ collection_names = [collection.name for collection in collections.collections]
 if collection_name not in collection_names:
     models.qdrant_client.create_collection(
         collection_name=collection_name,
-        vectors_config=VectorParams(size=1536, distance="Cosine") 
+        vectors_config=VectorParams(size=1024, distance="Cosine") 
     )
 
 qdrant_vector_store = QdrantVectorStore(
     client=models.qdrant_client,
     collection_name=collection_name,
-    embedding=models.te3_small
+    embedding=models.tuned_embeddings
 )
 
-qdrant_vector_store.add_documents(RCTS_split_docs)
+qdrant_vector_store.add_documents(semantic_tuned_split_docs)
 
 
 

models.py

@@ -5,9 +5,11 @@ from langchain.callbacks.tracers import LangChainTracer
 from langchain_huggingface.embeddings import HuggingFaceEmbeddings
 from langchain_experimental.text_splitter import SemanticChunker
 from langchain_openai.embeddings import OpenAIEmbeddings
-from langchain_community.vectorstores import Qdrant
+from langchain_qdrant import QdrantVectorStore, Qdrant
+from langchain.retrievers.contextual_compression import ContextualCompressionRetriever
 from qdrant_client import QdrantClient
 from langchain_text_splitters import RecursiveCharacterTextSplitter
+from langchain_cohere import CohereRerank
 import constants
 import os
 
@@ -18,7 +20,9 @@ os.environ["LANGCHAIN_ENDPOINT"] = constants.LANGCHAIN_ENDPOINT
 tracer = LangChainTracer()
 callback_manager = CallbackManager([tracer])
 
-qdrant_client = QdrantClient(url=constants.QDRANT_ENDPOINT, api_key=constants.QDRANT_API_KEY)
+########################
+### Chat Models      ###
+########################
 
 opus3 = ChatAnthropic(
     api_key=constants.ANTRHOPIC_API_KEY, 
@@ -65,22 +69,68 @@ gpt4o_mini = ChatOpenAI(
     callbacks=callback_manager
 )
 
+########################
+### Embedding Models ###
+########################
+
 basic_embeddings = HuggingFaceEmbeddings(model_name="snowflake/snowflake-arctic-embed-l")
-#hkunlp_instructor_large = HuggingFaceInstructEmbeddings(
-#    model_name = "hkunlp/instructor-large",
-#    query_instruction="Represent the query for retrieval: "
-#)
+
+tuned_embeddings = HuggingFaceEmbeddings(model_name="CoExperiences/snowflake-l-marketing-tuned")
 
 te3_small = OpenAIEmbeddings(api_key=constants.OPENAI_API_KEY, model="text-embedding-3-small")
 
+#######################
+### Text Splitters  ###
+#######################
+
 semanticChunker = SemanticChunker(
     te3_small,
     breakpoint_threshold_type="percentile"
 )
 
+semanticChunker_tuned = SemanticChunker(
+    tuned_embeddings,
+    breakpoint_threshold_type="percentile",
+    breakpoint_threshold_amount=85
+)
+
 RCTS = RecursiveCharacterTextSplitter(
     # Set a really small chunk size, just to show.
     chunk_size=500,
     chunk_overlap=25,
     length_function=len,
+)
+
+#######################
+###  Vector Stores  ###
+#######################
+
+qdrant_client = QdrantClient(url=constants.QDRANT_ENDPOINT, api_key=constants.QDRANT_API_KEY)
+
+semantic_Qdrant_vs = QdrantVectorStore(
+    client=qdrant_client,
+    collection_name="docs_from_ripped_urls",
+    embedding=te3_small
+)
+
+rcts_Qdrant_vs = QdrantVectorStore(
+    client=qdrant_client,
+    collection_name="docs_from_ripped_urls_recursive",
+    embedding=te3_small
+)
+
+semantic_tuned_Qdrant_vs = QdrantVectorStore(
+    client=qdrant_client,
+    collection_name="docs_from_ripped_urls_semantic_tuned",
+    embedding=tuned_embeddings
+)
+
+#######################
+###  Retrievers     ###
+#######################
+semantic_tuned_retriever = semantic_tuned_Qdrant_vs.as_retriever(search_kwargs={"k" : 10})
+
+compressor = CohereRerank(model="rerank-english-v3.0")
+compression_retriever = ContextualCompressionRetriever(
+    base_compressor=compressor, base_retriever=semantic_tuned_retriever
 )

multiagent.py

@@ -0,0 +1,336 @@
+#Change to requirements caller
+import sys
+import subprocess
+
+def run_pip_install():
+    packages = [
+        "langgraph",
+        "langchain",
+        "langchain_openai",
+        "langchain_experimental",
+        "qdrant-client",
+        "pymupdf",
+        "tiktoken",
+        "huggingface_hub",
+        "openai",
+        "tavily-python"
+    ]
+    
+    package_string = " ".join(packages)
+    
+    try:
+        subprocess.check_call([sys.executable, "-m", "pip", "install", "-qU"] + packages)
+        print("All required packages have been installed successfully.")
+    except subprocess.CalledProcessError:
+        print(f"Failed to install packages. Please run the following command manually:")
+        print(f"%pip install -qU {package_string}")
+        sys.exit(1)
+
+# Run pip install
+run_pip_install()
+
+import os
+import functools
+import operator
+from typing import Annotated, List, Tuple, Union, Dict, Optional
+from typing_extensions import TypedDict
+import uuid
+from pathlib import Path
+
+from langchain_core.tools import tool
+from langchain_core.messages import AIMessage, BaseMessage, HumanMessage
+from langchain_core.prompts import ChatPromptTemplate, MessagesPlaceholder
+from langchain_openai import ChatOpenAI
+from langchain.agents import AgentExecutor, create_openai_functions_agent
+from langchain.output_parsers.openai_functions import JsonOutputFunctionsParser
+from langchain_community.tools.tavily_search import TavilySearchResults
+from langchain_community.vectorstores import Qdrant
+from langchain.text_splitter import RecursiveCharacterTextSplitter
+from langchain_openai.embeddings import OpenAIEmbeddings
+from langgraph.graph import END, StateGraph
+from huggingface_hub import hf_hub_download
+
+# Environment setup
+OPENAI_API_KEY = os.environ.get("OPENAI_API_KEY")
+TAVILY_API_KEY = os.environ.get("TAVILY_API_KEY")
+
+if not OPENAI_API_KEY:
+    raise ValueError("OPENAI_API_KEY not found in environment variables")
+if not TAVILY_API_KEY:
+    raise ValueError("TAVILY_API_KEY not found in environment variables")
+
+os.environ["OPENAI_API_KEY"] = OPENAI_API_KEY
+os.environ["TAVILY_API_KEY"] = TAVILY_API_KEY
+
+# CHANGE TO HF DIRECTORY
+WORKING_DIRECTORY = Path("/tmp/content/data")
+WORKING_DIRECTORY.mkdir(parents=True, exist_ok=True)
+
+# Utility functions
+def create_random_subdirectory():
+    random_id = str(uuid.uuid4())[:8]
+    subdirectory_path = WORKING_DIRECTORY / random_id
+    subdirectory_path.mkdir(exist_ok=True)
+    return subdirectory_path
+
+def get_current_files():
+    try:
+        files = [f.relative_to(WORKING_DIRECTORY) for f in WORKING_DIRECTORY.rglob("*") if f.is_file()]
+        return "\n".join(str(f) for f in files) if files else "No files written."
+    except Exception:
+        return "Unable to retrieve current files."
+
+# Document loading change to upload in HF 
+def fetch_hbr_article():
+    pdf_path = hf_hub_download(repo_id="your-username/your-repo-name", filename="murthy-loneliness.pdf")
+    return PyMuPDFLoader(pdf_path).load()
+
+# Document processing
+def tiktoken_len(text):
+    tokens = tiktoken.encoding_for_model("gpt-4o-mini").encode(text)
+    return len(tokens)
+
+text_splitter = RecursiveCharacterTextSplitter(
+    chunk_size=300,
+    chunk_overlap=0,
+    length_function=tiktoken_len,
+)
+
+docs = fetch_hbr_article()
+split_chunks = text_splitter.split_documents(docs)
+
+# Embedding and vector store setup
+embedding_model = OpenAIEmbeddings(model="text-embedding-3-small")
+qdrant_vectorstore = Qdrant.from_documents(
+    split_chunks,
+    embedding_model,
+    location=":memory:",
+    collection_name="extending_context_window_llama_3",
+)
+qdrant_retriever = qdrant_vectorstore.as_retriever()
+
+# RAG setup
+RAG_PROMPT = """
+CONTEXT:
+{context}
+
+QUERY:
+{question}
+
+You are a helpful assistant. Use the available context to answer the question. If you can't answer the question, say you don't know.
+"""
+rag_prompt = ChatPromptTemplate.from_template(RAG_PROMPT)
+openai_chat_model = ChatOpenAI(model="gpt-4o-mini")
+
+rag_chain = (
+    {"context": itemgetter("question") | qdrant_retriever, "question": itemgetter("question")}
+    | rag_prompt | openai_chat_model | StrOutputParser()
+)
+
+# Tool definitions
+@tool
+def create_outline(points: List[str], file_name: str) -> str:
+    """Create and save an outline."""
+    with (WORKING_DIRECTORY / file_name).open("w") as file:
+        for i, point in enumerate(points):
+            file.write(f"{i + 1}. {point}\n")
+    return f"Outline saved to {file_name}"
+
+@tool
+def read_document(file_name: str, start: Optional[int] = None, end: Optional[int] = None) -> str:
+    """Read the specified document."""
+    with (WORKING_DIRECTORY / file_name).open("r") as file:
+        lines = file.readlines()
+    if start is not None:
+        start = 0
+    return "\n".join(lines[start:end])
+
+@tool
+def write_document(content: str, file_name: str) -> str:
+    """Create and save a text document."""
+    with (WORKING_DIRECTORY / file_name).open("w") as file:
+        file.write(content)
+    return f"Document saved to {file_name}"
+
+@tool
+def edit_document(file_name: str, inserts: Dict[int, str] = {}) -> str:
+    """Edit a document by inserting text at specific line numbers."""
+    with (WORKING_DIRECTORY / file_name).open("r") as file:
+        lines = file.readlines()
+    
+    sorted_inserts = sorted(inserts.items())
+    for line_number, text in sorted_inserts:
+        if 1 <= line_number <= len(lines) + 1:
+            lines.insert(line_number - 1, text + "\n")
+        else:
+            return f"Error: Line number {line_number} is out of range."
+    
+    with (WORKING_DIRECTORY / file_name).open("w") as file:
+        file.writelines(lines)
+    return f"Document edited and saved to {file_name}"
+
+@tool
+def retrieve_information(query: str):
+    """Use Retrieval Augmented Generation to retrieve information about the 'murthy-loneliness' paper."""
+    return rag_chain.invoke({"question": query})
+
+# Agent creation helpers
+def create_team_agent(llm, tools, system_prompt, agent_name, team_members):
+    return create_agent(
+        llm,
+        tools,
+        f"{system_prompt}\nBelow are files currently in your directory:\n{{current_files}}",
+        team_members
+    )
+
+def create_agent_node(agent, name):
+    return functools.partial(agent_node, agent=agent, name=name)
+
+def add_agent_to_graph(graph, agent_name, agent_node):
+    graph.add_node(agent_name, agent_node)
+    graph.add_edge(agent_name, "supervisor")
+
+def create_team_supervisor(llm, team_description, team_members):
+    return create_team_supervisor(
+        llm,
+        f"You are a supervisor tasked with managing a conversation between the"
+        f" following workers: {', '.join(team_members)}. {team_description}"
+        f" When all workers are finished, you must respond with FINISH.",
+        team_members
+    )
+
+def create_team_chain(graph, team_members):
+    return (
+        functools.partial(enter_chain, members=team_members)
+        | graph.compile()
+    )
+
+# LLM setup
+llm = ChatOpenAI(model="gpt-4-turbo")
+
+# Agent creation
+tavily_tool = TavilySearchResults(max_results=5)
+
+search_agent = create_team_agent(
+    llm,
+    [tavily_tool],
+    "You are a research assistant who can search for up-to-date info using the tavily search engine.",
+    "Search",
+    ["Search", "PaperInformationRetriever"]
+)
+
+research_agent = create_team_agent(
+    llm,
+    [retrieve_information],
+    "You are a research assistant who can provide specific information on the provided paper: 'murthy-loneliness.pdf'. You must only respond with information about the paper related to the request.",
+    "PaperInformationRetriever",
+    ["Search", "PaperInformationRetriever"]
+)
+
+doc_writer_agent = create_team_agent(
+    llm,
+    [write_document, edit_document, read_document],
+    "You are an expert writing technical social media posts.",
+    "DocWriter",
+    ["DocWriter", "NoteTaker", "CopyEditor", "VoiceEditor"]
+)
+
+note_taking_agent = create_team_agent(
+    llm,
+    [create_outline, read_document],
+    "You are an expert senior researcher tasked with writing a social media post outline and taking notes to craft a social media post.",
+    "NoteTaker",
+    ["DocWriter", "NoteTaker", "CopyEditor", "VoiceEditor"]
+)
+
+copy_editor_agent = create_team_agent(
+    llm,
+    [write_document, edit_document, read_document],
+    "You are an expert copy editor who focuses on fixing grammar, spelling, and tone issues.",
+    "CopyEditor",
+    ["DocWriter", "NoteTaker", "CopyEditor", "VoiceEditor"]
+)
+
+voice_editor_agent = create_team_agent(
+    llm,
+    [write_document, edit_document, read_document],
+    "You are an expert in crafting and refining the voice and tone of social media posts. You edit the document to ensure it has a consistent, professional, and engaging voice appropriate for social media platforms.",
+    "VoiceEditor",
+    ["DocWriter", "NoteTaker", "CopyEditor", "VoiceEditor"]
+)
+
+# Node creation
+search_node = create_agent_node(search_agent, "Search")
+research_node = create_agent_node(research_agent, "PaperInformationRetriever")
+doc_writing_node = create_agent_node(doc_writer_agent, "DocWriter")
+note_taking_node = create_agent_node(note_taking_agent, "NoteTaker")
+copy_editing_node = create_agent_node(copy_editor_agent, "CopyEditor")
+voice_node = create_agent_node(voice_editor_agent, "VoiceEditor")
+
+# Graph creation
+research_graph = StateGraph(ResearchTeamState)
+add_agent_to_graph(research_graph, "Search", search_node)
+add_agent_to_graph(research_graph, "PaperInformationRetriever", research_node)
+
+authoring_graph = StateGraph(DocWritingState)
+add_agent_to_graph(authoring_graph, "DocWriter", doc_writing_node)
+add_agent_to_graph(authoring_graph, "NoteTaker", note_taking_node)
+add_agent_to_graph(authoring_graph, "CopyEditor", copy_editing_node)
+add_agent_to_graph(authoring_graph, "VoiceEditor", voice_node)
+
+# Supervisor creation
+research_supervisor = create_team_supervisor(
+    llm,
+    "Given the following user request, determine the subject to be researched and respond with the worker to act next.",
+    ["Search", "PaperInformationRetriever"]
+)
+
+doc_writing_supervisor = create_team_supervisor(
+    llm,
+    "Given the following user request, determine which worker should act next. Each worker will perform a task and respond with their results and status.",
+    ["DocWriter", "NoteTaker", "CopyEditor", "VoiceEditor"]
+)
+
+# Graph compilation
+research_graph.add_node("supervisor", research_supervisor)
+research_graph.set_entry_point("supervisor")
+research_chain = create_team_chain(research_graph, research_graph.nodes)
+
+authoring_graph.add_node("supervisor", doc_writing_supervisor)
+authoring_graph.set_entry_point("supervisor")
+authoring_chain = create_team_chain(authoring_graph, authoring_graph.nodes)
+
+# Meta-supervisor setup
+super_graph = StateGraph(State)
+super_graph.add_node("Research team", get_last_message | research_chain | join_graph)
+super_graph.add_node("SocialMedia team", get_last_message | authoring_chain | join_graph)
+super_graph.add_node("supervisor", supervisor_node)
+
+super_graph.add_edge("Research team", "supervisor")
+super_graph.add_edge("SocialMedia team", "supervisor")
+super_graph.add_conditional_edges(
+    "supervisor",
+    lambda x: x["next"],
+    {
+        "SocialMedia team": "SocialMedia team",
+        "Research team": "Research team",
+        "FINISH": END,
+    },
+)
+super_graph.set_entry_point("supervisor")
+super_graph = super_graph.compile()
+
+# Example usage
+user_input = input("Enter your request for the social media post: ")
+
+for s in super_graph.stream(
+    {
+        "messages": [
+            HumanMessage(content=user_input)
+        ],
+    },
+    {"recursion_limit": 50},
+):
+    if "__end__" not in s:
+        print(s)
+        print("---")

requirements.txt

@@ -41,6 +41,7 @@ langchain-core==0.3.1
 langchain-openai==0.2.0
 langchain-qdrant==0.1.4
 langchain-text-splitters==0.3.0
+langgraph
 langsmith==0.1.121
 Lazify==0.4.0
 marshmallow==3.22.0
@@ -82,6 +83,7 @@ sniffio==1.3.1
 SQLAlchemy==2.0.35
 starlette==0.27.0
 syncer==2.0.3
+tavily-python
 tenacity==8.5.0
 tiktoken==0.7.0
 tomli==2.0.1

tuning/requirements.in

@@ -0,0 +1,14 @@
+langchain_openai
+langchain_huggingface
+langchain_core==0.2.38
+langchain
+langchain_community
+langchain-text-splitters
+faiss-cpu
+unstructured==0.15.7
+python-pptx==1.0.2
+nltk==3.9.1
+pyarrow
+sentence_transformers
+datasets
+ragas

tuning/requirements.txt

@@ -0,0 +1,412 @@
+#
+# This file is autogenerated by pip-compile with Python 3.11
+# by the following command:
+#
+#    pip-compile requirements.in
+#
+aiohappyeyeballs==2.4.3
+    # via aiohttp
+aiohttp==3.10.10
+    # via
+    #   datasets
+    #   fsspec
+    #   langchain
+    #   langchain-community
+aiosignal==1.3.1
+    # via aiohttp
+annotated-types==0.7.0
+    # via pydantic
+anyio==4.6.2.post1
+    # via
+    #   httpx
+    #   openai
+appdirs==1.4.4
+    # via ragas
+attrs==24.2.0
+    # via aiohttp
+backoff==2.2.1
+    # via unstructured
+beautifulsoup4==4.12.3
+    # via unstructured
+certifi==2024.8.30
+    # via
+    #   httpcore
+    #   httpx
+    #   requests
+cffi==1.17.1
+    # via cryptography
+chardet==5.2.0
+    # via unstructured
+charset-normalizer==3.4.0
+    # via requests
+click==8.1.7
+    # via nltk
+cryptography==43.0.1
+    # via unstructured-client
+dataclasses-json==0.6.7
+    # via
+    #   langchain-community
+    #   unstructured
+datasets==3.0.1
+    # via
+    #   -r requirements.in
+    #   ragas
+dill==0.3.8
+    # via
+    #   datasets
+    #   multiprocess
+distro==1.9.0
+    # via openai
+emoji==2.14.0
+    # via unstructured
+eval-type-backport==0.2.0
+    # via unstructured-client
+faiss-cpu==1.9.0
+    # via -r requirements.in
+filelock==3.16.1
+    # via
+    #   datasets
+    #   huggingface-hub
+    #   torch
+    #   transformers
+    #   triton
+filetype==1.2.0
+    # via unstructured
+frozenlist==1.4.1
+    # via
+    #   aiohttp
+    #   aiosignal
+fsspec[http]==2024.6.1
+    # via
+    #   datasets
+    #   huggingface-hub
+    #   torch
+greenlet==3.1.1
+    # via sqlalchemy
+h11==0.14.0
+    # via httpcore
+httpcore==1.0.6
+    # via httpx
+httpx==0.27.2
+    # via
+    #   langsmith
+    #   openai
+    #   unstructured-client
+huggingface-hub==0.26.0
+    # via
+    #   datasets
+    #   langchain-huggingface
+    #   sentence-transformers
+    #   tokenizers
+    #   transformers
+idna==3.10
+    # via
+    #   anyio
+    #   httpx
+    #   requests
+    #   yarl
+jinja2==3.1.4
+    # via torch
+jiter==0.6.1
+    # via openai
+joblib==1.4.2
+    # via
+    #   nltk
+    #   scikit-learn
+jsonpatch==1.33
+    # via langchain-core
+jsonpath-python==1.0.6
+    # via unstructured-client
+jsonpointer==3.0.0
+    # via jsonpatch
+langchain==0.2.16
+    # via
+    #   -r requirements.in
+    #   langchain-community
+    #   ragas
+langchain-community==0.2.16
+    # via
+    #   -r requirements.in
+    #   ragas
+langchain-core==0.2.38
+    # via
+    #   -r requirements.in
+    #   langchain
+    #   langchain-community
+    #   langchain-huggingface
+    #   langchain-openai
+    #   langchain-text-splitters
+    #   ragas
+langchain-huggingface==0.0.3
+    # via -r requirements.in
+langchain-openai==0.1.23
+    # via
+    #   -r requirements.in
+    #   ragas
+langchain-text-splitters==0.2.4
+    # via
+    #   -r requirements.in
+    #   langchain
+langdetect==1.0.9
+    # via unstructured
+langsmith==0.1.136
+    # via
+    #   langchain
+    #   langchain-community
+    #   langchain-core
+lxml==5.3.0
+    # via
+    #   python-pptx
+    #   unstructured
+markupsafe==3.0.2
+    # via jinja2
+marshmallow==3.23.0
+    # via dataclasses-json
+mpmath==1.3.0
+    # via sympy
+multidict==6.1.0
+    # via
+    #   aiohttp
+    #   yarl
+multiprocess==0.70.16
+    # via datasets
+mypy-extensions==1.0.0
+    # via typing-inspect
+nest-asyncio==1.6.0
+    # via
+    #   ragas
+    #   unstructured-client
+networkx==3.4.1
+    # via torch
+nltk==3.9.1
+    # via
+    #   -r requirements.in
+    #   unstructured
+numpy==1.26.4
+    # via
+    #   datasets
+    #   faiss-cpu
+    #   langchain
+    #   langchain-community
+    #   pandas
+    #   pyarrow
+    #   ragas
+    #   scikit-learn
+    #   scipy
+    #   transformers
+    #   unstructured
+nvidia-cublas-cu12==12.4.5.8
+    # via
+    #   nvidia-cudnn-cu12
+    #   nvidia-cusolver-cu12
+    #   torch
+nvidia-cuda-cupti-cu12==12.4.127
+    # via torch
+nvidia-cuda-nvrtc-cu12==12.4.127
+    # via torch
+nvidia-cuda-runtime-cu12==12.4.127
+    # via torch
+nvidia-cudnn-cu12==9.1.0.70
+    # via torch
+nvidia-cufft-cu12==11.2.1.3
+    # via torch
+nvidia-curand-cu12==10.3.5.147
+    # via torch
+nvidia-cusolver-cu12==11.6.1.9
+    # via torch
+nvidia-cusparse-cu12==12.3.1.170
+    # via
+    #   nvidia-cusolver-cu12
+    #   torch
+nvidia-nccl-cu12==2.21.5
+    # via torch
+nvidia-nvjitlink-cu12==12.4.127
+    # via
+    #   nvidia-cusolver-cu12
+    #   nvidia-cusparse-cu12
+    #   torch
+nvidia-nvtx-cu12==12.4.127
+    # via torch
+openai==1.52.0
+    # via
+    #   langchain-openai
+    #   ragas
+orjson==3.10.7
+    # via langsmith
+packaging==24.1
+    # via
+    #   datasets
+    #   faiss-cpu
+    #   huggingface-hub
+    #   langchain-core
+    #   marshmallow
+    #   transformers
+pandas==2.2.3
+    # via datasets
+pillow==11.0.0
+    # via
+    #   python-pptx
+    #   sentence-transformers
+propcache==0.2.0
+    # via yarl
+psutil==6.1.0
+    # via unstructured
+pyarrow==17.0.0
+    # via
+    #   -r requirements.in
+    #   datasets
+pycparser==2.22
+    # via cffi
+pydantic==2.9.2
+    # via
+    #   langchain
+    #   langchain-core
+    #   langsmith
+    #   openai
+    #   ragas
+    #   unstructured-client
+pydantic-core==2.23.4
+    # via pydantic
+pypdf==5.0.1
+    # via unstructured-client
+pysbd==0.3.4
+    # via ragas
+python-dateutil==2.8.2
+    # via
+    #   pandas
+    #   unstructured-client
+python-iso639==2024.4.27
+    # via unstructured
+python-magic==0.4.27
+    # via unstructured
+python-pptx==1.0.2
+    # via -r requirements.in
+pytz==2024.2
+    # via pandas
+pyyaml==6.0.2
+    # via
+    #   datasets
+    #   huggingface-hub
+    #   langchain
+    #   langchain-community
+    #   langchain-core
+    #   transformers
+ragas==0.2.1
+    # via -r requirements.in
+rapidfuzz==3.10.0
+    # via unstructured
+regex==2024.9.11
+    # via
+    #   nltk
+    #   tiktoken
+    #   transformers
+requests==2.32.3
+    # via
+    #   datasets
+    #   huggingface-hub
+    #   langchain
+    #   langchain-community
+    #   langsmith
+    #   requests-toolbelt
+    #   tiktoken
+    #   transformers
+    #   unstructured
+requests-toolbelt==1.0.0
+    # via
+    #   langsmith
+    #   unstructured-client
+safetensors==0.4.5
+    # via transformers
+scikit-learn==1.5.2
+    # via sentence-transformers
+scipy==1.14.1
+    # via
+    #   scikit-learn
+    #   sentence-transformers
+sentence-transformers==3.2.0
+    # via
+    #   -r requirements.in
+    #   langchain-huggingface
+six==1.16.0
+    # via
+    #   langdetect
+    #   python-dateutil
+sniffio==1.3.1
+    # via
+    #   anyio
+    #   httpx
+    #   openai
+soupsieve==2.6
+    # via beautifulsoup4
+sqlalchemy==2.0.36
+    # via
+    #   langchain
+    #   langchain-community
+sympy==1.13.1
+    # via torch
+tabulate==0.9.0
+    # via unstructured
+tenacity==8.5.0
+    # via
+    #   langchain
+    #   langchain-community
+    #   langchain-core
+threadpoolctl==3.5.0
+    # via scikit-learn
+tiktoken==0.8.0
+    # via
+    #   langchain-openai
+    #   ragas
+tokenizers==0.20.1
+    # via
+    #   langchain-huggingface
+    #   transformers
+torch==2.5.0
+    # via sentence-transformers
+tqdm==4.66.5
+    # via
+    #   datasets
+    #   huggingface-hub
+    #   nltk
+    #   openai
+    #   sentence-transformers
+    #   transformers
+    #   unstructured
+transformers==4.45.2
+    # via
+    #   langchain-huggingface
+    #   sentence-transformers
+triton==3.1.0
+    # via torch
+typing-extensions==4.12.2
+    # via
+    #   huggingface-hub
+    #   langchain-core
+    #   openai
+    #   pydantic
+    #   pydantic-core
+    #   python-pptx
+    #   sqlalchemy
+    #   torch
+    #   typing-inspect
+    #   unstructured
+typing-inspect==0.9.0
+    # via
+    #   dataclasses-json
+    #   unstructured-client
+tzdata==2024.2
+    # via pandas
+unstructured==0.15.7
+    # via -r requirements.in
+unstructured-client==0.26.1
+    # via unstructured
+urllib3==2.2.3
+    # via requests
+wrapt==1.16.0
+    # via unstructured
+xlsxwriter==3.2.0
+    # via python-pptx
+xxhash==3.5.0
+    # via datasets
+yarl==1.15.4
+    # via aiohttp