Upload 11 files

.gitattributes

@@ -32,3 +32,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+paper filter=lfs diff=lfs merge=lfs -text

README.md

@@ -1,12 +1,32 @@
----
-title: Paperchat
-emoji: 💩
-colorFrom: indigo
-colorTo: blue
-sdk: gradio
-sdk_version: 3.16.2
-app_file: app.py
-pinned: false
----
-
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+# PaperChat
+
+This repo is an implementation of a chatbot specifically focused on question answering over the [LangChain documentation](https://langchain.readthedocs.io/en/latest/).
+
+## 🚀 Important Links
+
+Website: [chat.langchain.dev](https://chat.langchain.dev)
+
+Hugging Face Spage: [huggingface.co/spaces/hwchase17/chat-langchain](https://huggingface.co/spaces/hwchase17/chat-langchain)
+
+Blog Post: [blog.langchain.dev/langchain-chat/](https://blog.langchain.dev/langchain-chat/)
+
+## 📚 Technical description
+
+There are two components: ingestion and question-answering.
+
+Ingestion has the following steps:
+
+1. Pull html from documentation site
+2. Parse html with BeautifulSoup
+3. Split documents with LangChain's [TextSplitter](https://langchain.readthedocs.io/en/latest/modules/utils/combine_docs_examples/textsplitter.html)
+4. Create a vectorstore of embeddings, using LangChain's [vectorstore wrapper](https://langchain.readthedocs.io/en/latest/modules/utils/combine_docs_examples/vectorstores.html) (with OpenAI's embeddings and Weaviate's vectorstore)
+
+Question-Answering has the following steps:
+
+1. Given the chat history and new user input, determine what a standalone question would be (using GPT-3)
+2. Given that standalone question, look up relevant documents from the vectorstore
+3. Pass the standalone question and relevant documents to GPT-3 to generate a final answer
+
+## 🧠 How to Extend to your documentation
+
+Coming soon.

app.py

@@ -0,0 +1,144 @@
+import datetime
+import os
+from langchain.chains import VectorDBQAWithSourcesChain
+import gradio as gr
+import langchain
+import weaviate
+from langchain.vectorstores import Weaviate
+import faiss
+import pickle
+from langchain import OpenAI
+from arxiv import get_paper
+from ingest_faiss import create_vector_store
+
+def get_vectorstore(suffix):
+    index = faiss.read_index(f"{suffix}/docs.index")
+    with open(f"{suffix}/faiss_store.pkl", "rb") as f:
+        store = pickle.load(f)
+    store.index = index
+    return store
+
+def set_openai_api_key(api_key, agent):
+    if api_key:
+        os.environ["OPENAI_API_KEY"] = api_key
+        vectorstore = get_vectorstore()
+        qa_chain = VectorDBQAWithSourcesChain.from_llm(llm=OpenAI(temperature=0), vectorstore=vectorstore)
+        os.environ["OPENAI_API_KEY"] = ""
+        return qa_chain
+
+def download_paper_and_embed(paper_arxiv_url, api_key):
+    if paper_arxiv_url and api_key:
+        paper_text = get_paper(paper_arxiv_url)
+        if 'abs' in paper_arxiv_url:
+            eprint_url = paper_arxiv_url.replace("https://arxiv.org/abs/", "https://arxiv.org/e-print/")
+        elif 'pdf' in paper_arxiv_url:
+            eprint_url = paper_arxiv_url.replace("https://arxiv.org/pdf/", "https://arxiv.org/e-print/")
+        else:
+            raise ValueError("Invalid arXiv URL")
+        suffix = 'paper-dir/' + eprint_url.replace("https://arxiv.org/e-print/", "")
+        if not os.path.exists(suffix + "/docs.index"):
+            create_vector_store(suffix, paper_text)
+
+        os.environ["OPENAI_API_KEY"] = api_key
+        vectorstore = get_vectorstore(suffix)
+        qa_chain = VectorDBQAWithSourcesChain.from_llm(llm=OpenAI(temperature=0), vectorstore=vectorstore)
+        os.environ["OPENAI_API_KEY"] = ""
+        return qa_chain
+
+chain = None
+
+def chat(inp, history, paper_arxiv_url, api_key, agent):
+    global chain
+    if history is None:
+        chain = download_paper_and_embed(paper_arxiv_url, api_key)
+    history = history or []
+    # if agent is None:
+    #     history.append((inp, "Please paste your OpenAI key to use"))
+    #     return history, history
+    print("\n==== date/time: " + str(datetime.datetime.now()) + " ====")
+    print("inp: " + inp)
+    history = history or []
+    agent = chain
+    output = agent({"question": inp})
+    answer = output["answer"]
+    sources = output["sources"]
+    history.append((inp, answer))
+    history.append(("Sources?", sources))
+    print(history)
+    return history, history
+
+block = gr.Blocks(css=".gradio-container {background-color: lightgray}")
+
+with block:
+    state = gr.State()
+    agent_state = gr.State()
+    with gr.Row():
+        gr.Markdown("<h3><center>PaperChat</center></h3>")
+
+        paper_arxiv_url = gr.Textbox(
+            placeholder="Paste the URL of the paper about which you want to ask a question",
+            show_label=False,
+            lines=1,
+            type="url",
+        )
+
+        openai_api_key_textbox = gr.Textbox(
+            placeholder="Paste your OpenAI API key (sk-...)",
+            show_label=False,
+            lines=1,
+            type="password",
+        )
+
+        # # button to download paper
+        # download_paper_button = gr.Button(
+        #     value="Download paper and make embeddings",
+        #     variant="secondary",
+        # ).click(
+        #     download_paper_and_embed,
+        #     inputs=[paper_arxiv_url, openai_api_key_textbox, agent_state],
+        #     outputs=[agent_state],
+        # )
+
+    chatbot = gr.Chatbot()
+
+    with gr.Row():
+        message = gr.Textbox(
+            label="What's your question?",
+            placeholder="What's the answer to life, the universe, and everything?",
+            lines=1,
+        )
+        submit = gr.Button(value="Send", variant="secondary").style(full_width=False)
+
+    # gr.Examples(
+    #     examples=[
+    #         "What are agents?",
+    #         "How do I summarize a long document?",
+    #         "What types of memory exist?",
+    #     ],
+    #     inputs=message,
+    # )
+
+    gr.HTML(
+        """This app demonstrates question-answering on any given arxiv paper"""
+    )
+
+    gr.HTML(
+        "<center>Powered by <a href='https://github.com/hwchase17/langchain'>LangChain 🦜️🔗</a></center>"
+    )
+
+    submit.click(chat, inputs=[message, state, paper_arxiv_url, openai_api_key_textbox, agent_state], outputs=[chatbot, state])
+    message.submit(chat, inputs=[message, state, paper_arxiv_url, openai_api_key_textbox, agent_state], outputs=[chatbot, state])
+
+    # paper_arxiv_url.change(
+    #     download_paper_and_embed,
+    #     inputs=[paper_arxiv_url, agent_state],
+    #     outputs=[agent_state],
+    # )
+
+    # openai_api_key_textbox.change(
+    #     set_openai_api_key,
+    #     inputs=[openai_api_key_textbox, agent_state],
+    #     outputs=[agent_state],
+    # )
+
+block.launch(debug=True)

arxiv.py

@@ -0,0 +1,55 @@
+import requests
+from subprocess import call
+import os
+from pylatexenc.latex2text import LatexNodes2Text
+
+def get_paper(paper_url):
+    if 'abs' in paper_url:
+        eprint_url = paper_url.replace("https://arxiv.org/abs/", "https://arxiv.org/e-print/")
+    elif 'pdf' in paper_url:
+        eprint_url = paper_url.replace("https://arxiv.org/pdf/", "https://arxiv.org/e-print/")
+    else:
+        raise ValueError("Invalid arXiv URL")
+
+    suffix = 'paper-dir/' + eprint_url.replace("https://arxiv.org/e-print/", "")
+
+    # check if the directory exists
+    if os.path.exists(suffix):
+        print("Paper already downloaded, skipping download")
+    else:
+        print("Downloading paper")
+        r = requests.get(eprint_url)
+
+        with open("paper", "wb") as f:
+            f.write(r.content)
+
+        # unzip gzipped tar file to new directory
+        call(["mkdir", suffix])
+        call(["tar", "-xzf", "paper", "-C", suffix])
+
+    # get the list of all .tex files in the directory
+    tex_files = [f for f in os.listdir(suffix) if f.endswith('.tex')]
+    # remove math_commands.tex from tex_files if it exists
+    if 'math_commands.tex' in tex_files:
+        tex_files.remove('math_commands.tex')
+    if len(tex_files) == 1:
+        # read the main tex file
+        with open(f'{suffix}/{tex_files[0]}', 'r') as f:
+            paper_tex = f.read()
+    elif len(tex_files) == 0:
+        raise ValueError("No .tex files found in the paper")
+    else:
+        raise ValueError("More than one .tex file found in the paper")
+    
+    # convert latex to text
+    paper_text = LatexNodes2Text().latex_to_text(paper_tex)
+
+    with open(f"{suffix}/main.txt", 'w') as f:
+        f.write(paper_text)
+    
+    return paper_text
+
+if __name__=="__main__":
+    paper_url = "https://arxiv.org/abs/2206.08896"
+    paper_text = get_paper(paper_url)
+    print(paper_text)

chain.py

@@ -0,0 +1,128 @@
+import json
+import os
+import pathlib
+from typing import Dict, List, Tuple
+
+import weaviate
+from langchain import OpenAI, PromptTemplate
+from langchain.chains import LLMChain
+from langchain.chains.base import Chain
+from langchain.chains.combine_documents.base import BaseCombineDocumentsChain
+from langchain.chains.conversation.memory import ConversationBufferMemory
+from langchain.chains.question_answering import load_qa_chain
+from langchain.embeddings import OpenAIEmbeddings
+from langchain.prompts import FewShotPromptTemplate, PromptTemplate
+from langchain.prompts.example_selector import \
+    SemanticSimilarityExampleSelector
+from langchain.vectorstores import FAISS, Weaviate
+from pydantic import BaseModel
+
+
+class CustomChain(Chain, BaseModel):
+
+    vstore: Weaviate
+    chain: BaseCombineDocumentsChain
+    key_word_extractor: Chain
+
+    @property
+    def input_keys(self) -> List[str]:
+        return ["question"]
+
+    @property
+    def output_keys(self) -> List[str]:
+        return ["answer"]
+
+    def _call(self, inputs: Dict[str, str]) -> Dict[str, str]:
+        question = inputs["question"]
+        chat_history_str = _get_chat_history(inputs["chat_history"])
+        if chat_history_str:
+            new_question = self.key_word_extractor.run(
+                question=question, chat_history=chat_history_str
+            )
+        else:
+            new_question = question
+        print(new_question)
+        docs = self.vstore.similarity_search(new_question, k=4)
+        new_inputs = inputs.copy()
+        new_inputs["question"] = new_question
+        new_inputs["chat_history"] = chat_history_str
+        answer, _ = self.chain.combine_docs(docs, **new_inputs)
+        return {"answer": answer}
+
+
+def get_new_chain1(vectorstore) -> Chain:
+    WEAVIATE_URL = os.environ["WEAVIATE_URL"]
+    client = weaviate.Client(
+        url=WEAVIATE_URL,
+        additional_headers={"X-OpenAI-Api-Key": os.environ["OPENAI_API_KEY"]},
+    )
+
+    _eg_template = """## Example:
+
+    Chat History:
+    {chat_history}
+    Follow Up Input: {question}
+    Standalone question: {answer}"""
+    _eg_prompt = PromptTemplate(
+        template=_eg_template,
+        input_variables=["chat_history", "question", "answer"],
+    )
+
+    _prefix = """Given the following conversation and a follow up question, rephrase the follow up question to be a standalone question. You should assume that the question is related to LangChain."""
+    _suffix = """## Example:
+
+    Chat History:
+    {chat_history}
+    Follow Up Input: {question}
+    Standalone question:"""
+    eg_store = Weaviate(
+        client,
+        "Rephrase",
+        "content",
+        attributes=["question", "answer", "chat_history"],
+    )
+    example_selector = SemanticSimilarityExampleSelector(vectorstore=eg_store, k=4)
+    prompt = FewShotPromptTemplate(
+        prefix=_prefix,
+        suffix=_suffix,
+        example_selector=example_selector,
+        example_prompt=_eg_prompt,
+        input_variables=["question", "chat_history"],
+    )
+    llm = OpenAI(temperature=0, model_name="text-davinci-003")
+    key_word_extractor = LLMChain(llm=llm, prompt=prompt)
+
+    EXAMPLE_PROMPT = PromptTemplate(
+        template=">Example:\nContent:\n---------\n{page_content}\n----------\nSource: {source}",
+        input_variables=["page_content", "source"],
+    )
+    template = """You are an AI assistant for the open source library LangChain. The documentation is located at https://langchain.readthedocs.io.
+You are given the following extracted parts of a long document and a question. Provide a conversational answer with a hyperlink to the documentation.
+You should only use hyperlinks that are explicitly listed as a source in the context. Do NOT make up a hyperlink that is not listed.
+If the question includes a request for code, provide a code block directly from the documentation.
+If you don't know the answer, just say "Hmm, I'm not sure." Don't try to make up an answer.
+If the question is not about LangChain, politely inform them that you are tuned to only answer questions about LangChain.
+Question: {question}
+=========
+{context}
+=========
+Answer in Markdown:"""
+    PROMPT = PromptTemplate(template=template, input_variables=["question", "context"])
+    doc_chain = load_qa_chain(
+        OpenAI(temperature=0, model_name="text-davinci-003", max_tokens=-1),
+        chain_type="stuff",
+        prompt=PROMPT,
+        document_prompt=EXAMPLE_PROMPT,
+    )
+    return CustomChain(
+        chain=doc_chain, vstore=vectorstore, key_word_extractor=key_word_extractor
+    )
+
+
+def _get_chat_history(chat_history: List[Tuple[str, str]]):
+    buffer = ""
+    for human_s, ai_s in chat_history:
+        human = f"Human: " + human_s
+        ai = f"Assistant: " + ai_s
+        buffer += "\n" + "\n".join([human, ai])
+    return buffer

ingest.py

@@ -0,0 +1,99 @@
+"""Load html from files, clean up, split, ingest into Weaviate."""
+import os
+from pathlib import Path
+
+import weaviate
+from bs4 import BeautifulSoup
+from langchain.text_splitter import CharacterTextSplitter
+
+# def clean_data(data):
+#     soup = BeautifulSoup(data)
+#     text = soup.find_all("main", {"id": "main-content"})[0].get_text()
+#     return "\n".join([t for t in text.split("\n") if t])
+
+# docs = []
+# metadatas = []
+# for p in Path("langchain.readthedocs.io/en/latest/").rglob("*"):
+#     if p.is_dir():
+#         continue
+#     with open(p) as f:
+#         docs.append(clean_data(f.read()))
+#         metadatas.append({"source": p})
+
+with open('paper-dir/main.txt') as f:
+    paper_text = f.read()
+
+docs = paper_text.split("§")
+# metadatas is the first word that comes after the section symbol
+metadatas = [doc.split(" ")[0] for doc in docs]
+
+text_splitter = CharacterTextSplitter(
+    separator="\n",
+    chunk_size=1000,
+    chunk_overlap=200,
+    length_function=len,
+)
+
+documents = text_splitter.create_documents(docs, metadatas=metadatas)
+
+
+WEAVIATE_URL = os.environ["WEAVIATE_URL"]
+client = weaviate.Client(
+    url=WEAVIATE_URL,
+    additional_headers={"X-OpenAI-Api-Key": os.environ["OPENAI_API_KEY"]},
+)
+
+client.schema.delete_class("Paragraph")
+client.schema.get()
+schema = {
+    "classes": [
+        {
+            "class": "Paragraph",
+            "description": "A written paragraph",
+            "vectorizer": "text2vec-openai",
+            "moduleConfig": {
+                "text2vec-openai": {
+                    "model": "ada",
+                    "modelVersion": "002",
+                    "type": "text",
+                }
+            },
+            "properties": [
+                {
+                    "dataType": ["text"],
+                    "description": "The content of the paragraph",
+                    "moduleConfig": {
+                        "text2vec-openai": {
+                            "skip": False,
+                            "vectorizePropertyName": False,
+                        }
+                    },
+                    "name": "content",
+                },
+                {
+                    "dataType": ["text"],
+                    "description": "The link",
+                    "moduleConfig": {
+                        "text2vec-openai": {
+                            "skip": True,
+                            "vectorizePropertyName": False,
+                        }
+                    },
+                    "name": "source",
+                },
+            ],
+        },
+    ]
+}
+
+client.schema.create(schema)
+
+with client.batch as batch:
+    for text in documents:
+        batch.add_data_object(
+            {
+                "content": text.page_content,
+                "source": str(text.metadata["source"])
+            },
+            "Paragraph",
+        )

ingest.sh

@@ -0,0 +1,6 @@
+# Bash script to ingest data
+# This involves scraping the data from the web and then cleaning up and putting in Weaviate.
+!set -eu
+wget -r -A.html https://langchain.readthedocs.io/en/latest/
+python3 ingest.py
+python3 ingest_examples.py

ingest_examples.py

@@ -0,0 +1,219 @@
+"""Ingest examples into Weaviate."""
+import os
+from pathlib import Path
+
+import weaviate
+
+WEAVIATE_URL = os.environ["WEAVIATE_URL"]
+client = weaviate.Client(
+    url=WEAVIATE_URL,
+    additional_headers={"X-OpenAI-Api-Key": os.environ["OPENAI_API_KEY"]},
+)
+
+client.schema.delete_class("Rephrase")
+client.schema.delete_class("QA")
+client.schema.get()
+schema = {
+    "classes": [
+        {
+            "class": "Rephrase",
+            "description": "Rephrase Examples",
+            "vectorizer": "text2vec-openai",
+            "moduleConfig": {
+                "text2vec-openai": {
+                    "model": "ada",
+                    "modelVersion": "002",
+                    "type": "text",
+                }
+            },
+            "properties": [
+                {
+                    "dataType": ["text"],
+                    "moduleConfig": {
+                        "text2vec-openai": {
+                            "skip": False,
+                            "vectorizePropertyName": False,
+                        }
+                    },
+                    "name": "content",
+                },
+                {
+                    "dataType": ["text"],
+                    "description": "The link",
+                    "moduleConfig": {
+                        "text2vec-openai": {
+                            "skip": True,
+                            "vectorizePropertyName": False,
+                        }
+                    },
+                    "name": "question",
+                },
+                {
+                    "dataType": ["text"],
+                    "description": "The link",
+                    "moduleConfig": {
+                        "text2vec-openai": {
+                            "skip": True,
+                            "vectorizePropertyName": False,
+                        }
+                    },
+                    "name": "answer",
+                },
+                {
+                    "dataType": ["text"],
+                    "description": "The link",
+                    "moduleConfig": {
+                        "text2vec-openai": {
+                            "skip": True,
+                            "vectorizePropertyName": False,
+                        }
+                    },
+                    "name": "chat_history",
+                },
+            ],
+        },
+    ]
+}
+
+client.schema.create(schema)
+
+documents = [
+    {
+        "question": "how do i load those?",
+        "chat_history": "Human: What types of memory exist?\nAssistant: \n\nThere are a few different types of memory: Buffer, Summary, and Conversational Memory.",
+        "answer": "How do I load Buffer, Summary, and Conversational Memory",
+    },
+    {
+        "question": "how do i install this package?",
+        "chat_history": "",
+        "answer": "How do I install langchain?",
+    },
+    {
+        "question": "how do I set serpapi_api_key?",
+        "chat_history": "Human: can you write me a code snippet for that?\nAssistant: \n\nYes, you can create an Agent with a custom LLMChain in LangChain. Here is a [link](https://langchain.readthedocs.io/en/latest/modules/agents/examples/custom_agent.html) to the documentation that provides a code snippet for creating a custom Agent.",
+        "answer": "How do I set the serpapi_api_key?",
+    },
+    {
+        "question": "What are some methods for data augmented generation?",
+        "chat_history": "Human: List all methods of an Agent class please\nAssistant: \n\nTo answer your question, you can find a list of all the methods of the Agent class in the [API reference documentation](https://langchain.readthedocs.io/en/latest/modules/agents/reference.html).",
+        "answer": "What are some methods for data augmented generation?",
+    },
+    {
+        "question": "can you write me a code snippet for that?",
+        "chat_history": "Human: how do I create an agent with custom LLMChain?\nAssistant: \n\nTo create an Agent with a custom LLMChain in LangChain, you can use the [Custom Agent example](https://langchain.readthedocs.io/en/latest/modules/agents/examples/custom_agent.html). This example shows how to create a custom LLMChain and use an existing Agent class to parse the output. For more information on Agents and Tools, check out the [Key Concepts](https://langchain.readthedocs.io/en/latest/modules/agents/key_concepts.html) documentation.",
+        "answer": "Can you provide a code snippet for creating an Agent with a custom LLMChain?",
+    },
+]
+from langchain.prompts.example_selector.semantic_similarity import \
+    sorted_values
+
+for d in documents:
+    d["content"] = " ".join(sorted_values(d))
+with client.batch as batch:
+    for text in documents:
+        batch.add_data_object(
+            text,
+            "Rephrase",
+        )
+
+client.schema.get()
+schema = {
+    "classes": [
+        {
+            "class": "QA",
+            "description": "Rephrase Examples",
+            "vectorizer": "text2vec-openai",
+            "moduleConfig": {
+                "text2vec-openai": {
+                    "model": "ada",
+                    "modelVersion": "002",
+                    "type": "text",
+                }
+            },
+            "properties": [
+                {
+                    "dataType": ["text"],
+                    "moduleConfig": {
+                        "text2vec-openai": {
+                            "skip": False,
+                            "vectorizePropertyName": False,
+                        }
+                    },
+                    "name": "content",
+                },
+                {
+                    "dataType": ["text"],
+                    "description": "The link",
+                    "moduleConfig": {
+                        "text2vec-openai": {
+                            "skip": True,
+                            "vectorizePropertyName": False,
+                        }
+                    },
+                    "name": "question",
+                },
+                {
+                    "dataType": ["text"],
+                    "description": "The link",
+                    "moduleConfig": {
+                        "text2vec-openai": {
+                            "skip": True,
+                            "vectorizePropertyName": False,
+                        }
+                    },
+                    "name": "answer",
+                },
+                {
+                    "dataType": ["text"],
+                    "description": "The link",
+                    "moduleConfig": {
+                        "text2vec-openai": {
+                            "skip": True,
+                            "vectorizePropertyName": False,
+                        }
+                    },
+                    "name": "summaries",
+                },
+                {
+                    "dataType": ["text"],
+                    "description": "The link",
+                    "moduleConfig": {
+                        "text2vec-openai": {
+                            "skip": True,
+                            "vectorizePropertyName": False,
+                        }
+                    },
+                    "name": "sources",
+                },
+            ],
+        },
+    ]
+}
+
+client.schema.create(schema)
+
+documents = [
+    {
+        "question": "how do i install langchain?",
+        "answer": "```pip install langchain```",
+        "summaries": ">Example:\nContent:\n---------\nYou can pip install langchain package by running 'pip install langchain'\n----------\nSource: foo.html",
+        "sources": "foo.html",
+    },
+    {
+        "question": "how do i import an openai LLM?",
+        "answer": "```from langchain.llm import OpenAI```",
+        "summaries": ">Example:\nContent:\n---------\nyou can import the open ai wrapper (OpenAI) from the langchain.llm module\n----------\nSource: bar.html",
+        "sources": "bar.html",
+    },
+]
+from langchain.prompts.example_selector.semantic_similarity import \
+    sorted_values
+
+for d in documents:
+    d["content"] = " ".join(sorted_values(d))
+with client.batch as batch:
+    for text in documents:
+        batch.add_data_object(
+            text,
+            "QA",
+        )

ingest_faiss.py

@@ -0,0 +1,41 @@
+from pathlib import Path
+from langchain.text_splitter import CharacterTextSplitter
+import faiss
+from langchain.vectorstores import FAISS
+from langchain.embeddings import OpenAIEmbeddings
+import pickle
+
+def create_vector_store(suffix, paper_text):
+    # with open('paper-dir/main.txt') as f:
+    #     paper_text = f.read()
+
+    split_chars = ["§", "§.§"]
+    data = []
+    for c in split_chars:
+        paper_text = paper_text.replace(c, "§")
+    data = paper_text.split("§")
+
+    # metadatas is the rest of the text on the same line as the section symbol
+    sources = []
+    for d in data:
+        sources.append(d.split("\n")[0].strip())
+        # data = [d.split("\n")[1:] for d in data]
+
+    sources[0] = "Beginning of paper"
+
+    # Here we split the documents, as needed, into smaller chunks.
+    # We do this due to the context limits of the LLMs.
+    text_splitter = CharacterTextSplitter(chunk_size=1500, separator="\n")
+    docs = []
+    metadatas = []
+    for i, d in enumerate(data):
+        splits = text_splitter.split_text(d)
+        docs.extend(splits)
+        metadatas.extend([{"source": sources[i]}] * len(splits))
+
+    # Here we create a vector store from the documents and save it to disk.
+    store = FAISS.from_texts(docs, OpenAIEmbeddings(), metadatas=metadatas)
+    faiss.write_index(store.index, f"{suffix}/docs.index")
+    store.index = None
+    with open(f"{suffix}/faiss_store.pkl", "wb") as f:
+        pickle.dump(store, f)

paper

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9aeb23e3ec5eb22876c9c3b823841a82706cabe5aeacbe1783686a1db8aa0285
+size 10862082

qa.py

@@ -0,0 +1,26 @@
+"""Ask a question to the notion database."""
+import faiss
+from langchain import OpenAI
+from langchain.chains import VectorDBQAWithSourcesChain
+import pickle
+import argparse
+
+parser = argparse.ArgumentParser(description='Ask a question about the paper')
+parser.add_argument('question', type=str, help='The question to ask about the paper')
+args = parser.parse_args()
+
+# Load the LangChain.
+index = faiss.read_index("docs.index")
+
+with open("faiss_store.pkl", "rb") as f:
+    store = pickle.load(f)
+
+store.index = index
+chain = VectorDBQAWithSourcesChain.from_llm(llm=OpenAI(temperature=0), vectorstore=store)
+result = chain({"question": args.question})
+print(f"Answer: {result['answer']}")
+sources = result["sources"].split(", ")
+sources = [s.title() for s in sources]
+
+import code
+code.interact(local=locals())

requirements.txt

@@ -0,0 +1,10 @@
+langchain==0.0.64
+beautifulsoup4
+weaviate-client
+openai
+black
+isort
+Flask
+transformers
+gradio
+pylatexenc