Update pages/bot.py

pages/bot.py

@@ -1,36 +1,100 @@
 import streamlit as st
-from transformers import pipeline
-import datetime
-import pandas as pd
-from pathlib import Path   
-# to-do import upload
+from langchain.embeddings import HuggingFaceInstructEmbeddings
+from langchain.vectorstores import FAISS
+from langchain.text_splitter import CharacterTextSplitter
+from langchain.document_loaders import DirectoryLoader, PyPDFLoader
+import os
+from PyPDF2 import PdfReader
+from langchain.chains import RetrievalQAWithSourcesChain
+from langchain.memory import ConversationBufferMemory
+from langchain.chains import ConversationalRetrievalChain
+#from htmlTemplates import css, bot_template, user_template
+from langchain.llms import HuggingFaceHub
+from dotenv import load_dotenv
+###########
+#pip install faiss-cpu
+#pip install langchain
+#pip install pypdf
+#pip tiktoken
+#pip install InstructorEmbedding
+###############
 
+# PDF in String umwandeln
+def get_pdf_text(folder_path):
+    text = ""
+    # Durchsuche alle Dateien im angegebenen Verzeichnis
+    for filename in os.listdir(folder_path):
+        filepath = os.path.join(folder_path, filename)
 
-st.markdown("# Chatbot")
-st.sidebar.markdown("# Chatbot")
+        # Überprüfe, ob die Datei die Erweiterung ".pdf" hat
+        if os.path.isfile(filepath) and filename.lower().endswith(".pdf"):
+            pdf_reader = PdfReader(filepath)
+            for page in pdf_reader.pages:
+                text += page.extract_text()
+            #text += '\n'
 
-uploaded_file = '' # PLACEHOLDER
+    return text
 
-# Display file content
-file_content = uploaded_file.read()
-st.write("Dateiinhalt:")
-st.code(file_content)
+#Chunks erstellen
+def get_text_chunks(text):
+    #Arbeitsweise Textsplitter definieren
+    text_splitter = CharacterTextSplitter(
+        separator="\n",
+        chunk_size=1000,
+        chunk_overlap=200,
+        length_function=len
+    )
+    chunks = text_splitter.split_text(text)
+    return chunks
+
+# nur zum Anlegen des lokalen Verzeichnisses "Store" und speichern der Vektor-Datenbank
+def create_vectorstore_and_store(text_chunks):
+    embeddings = HuggingFaceInstructEmbeddings(model_name="hkunlp/instructor-base")
+    # Initiate Faiss DB
+    vectorstoreDB = FAISS.from_texts(texts=text_chunks,embedding=embeddings)#texts=text_chunks,
+    ###
+    ### --> danach soll das PDF-Verzeichnis gelöscht werden, bzw. Datein verschieben, weil beim nächsten Upload 
+    ###
+    # Verzeichnis in dem die VektorDB gespeichert werden soll
+    save_directory = "Store"
+    #VektorDB lokal speichern
+    vectorstoreDB.save_local(save_directory)
+    print(vectorstoreDB)
+    return None
+    
+########
 
-# User input for question
-user_question = st.text_input("Stellen Sie eine Frage zum hochgeladenen PDF:")
+def get_vectorstore():
+    embeddings = HuggingFaceInstructEmbeddings(model_name="hkunlp/instructor-base")
+    #Abruf lokaler Vektordatenbank
+    save_directory = "Store"
+    vectorstoreDB = FAISS.load_local(save_directory, embeddings)
+    return vectorstoreDB
 
 
-    # Perform Hugging Face task (e.g., question answering)
-if user_question:
-    question_answering = pipeline(
-        "question-answering",
-        model="deepset/gelectra-base-germanquad-distilled",
-        tokenizer="deepset/gelectra-base-germanquad-distilled"
+def main():
+    load_dotenv()
+    user_question = st.text_area("Eingabe:")
+    folder_path = './PDFs'
+    pdf_text = get_pdf_text(folder_path)
+    text_chunks = get_text_chunks(pdf_text)
+    create_vectorstore_and_store(text_chunks)
+    
+    retriever=get_vectorstore().as_retriever()
+    retrieved_docs=retriever.invoke(
+    user_question
     )
+    if user_question:
+        st.text(retrieved_docs[0].page_content)
+          # bei incoming pdf
+
+    #vectorstore_DB=get_vectorstore()        # bei Abfrage durch Chatbot
+    #print(get_vectorstore().similarity_search_with_score("stelle")) # zeigt an ob Vektordatenbank gefüllt ist
+    
+    #print(get_conversation_chain(get_vectorstore()))
+
+
 
-    # Get answer to the user's question
-    answer = question_answering(question=user_question, context=file_content)
 
-    # Display the answer to the user's question
-    st.write(f"Antwort auf die Frage '{user_question}': {answer['answer']}")
-    st.write("Confidence Score:", answer['score'])
+if __name__ == '__main__':
+    main()