Update app.py

app.py

@@ -1,7 +1,9 @@
-import streamlit as st
-import re
+import os
 import pickle
+from nltk.tokenize import sent_tokenize
+import nltk
 from PyPDF2 import PdfReader
+import streamlit as st
 from streamlit_extras.add_vertical_space import add_vertical_space
 from langchain.text_splitter import RecursiveCharacterTextSplitter
 from langchain.embeddings.openai import OpenAIEmbeddings
@@ -9,13 +11,14 @@ from langchain.vectorstores import FAISS
 from langchain.llms import OpenAI
 from langchain.chains.question_answering import load_qa_chain
 from langchain.callbacks import get_openai_callback
-import os
+
+nltk.download('punkt')
 
 # Sidebar contents
 with st.sidebar:
     st.title(':orange_book: BinDoc GmbH')
     
-    # API key input (this will not display the entered text)
+    # API key input
     api_key = st.text_input('Enter your OpenAI API Key:', type='password')
     
     if api_key:
@@ -23,22 +26,15 @@ with st.sidebar:
     else:
         st.warning('API key is required to proceed.')
 
-    st.markdown(
-        "Experience the future of document interaction with the revolutionary"
-    )
-
+    st.markdown("Experience the future of document interaction with the revolutionary")
     st.markdown("**BinDocs Chat App**.")
-
     st.markdown("Harnessing the power of a Large Language Model and AI technology,")
-
     st.markdown("this innovative platform redefines PDF engagement,")
-
     st.markdown("enabling dynamic conversations that bridge the gap between")
     st.markdown("human and machine intelligence.")
 
     add_vertical_space(3)  # Add more vertical space between text blocks
     st.write('Made with ❤️ by BinDoc GmbH')
-    
 
 def load_pdf(file_path):
     pdf_reader = PdfReader(file_path)
@@ -49,28 +45,39 @@ def load_pdf(file_path):
             chunks.append(text)
     
     store_name = file_path.name[:-4]
-
+    
     if os.path.exists(f"{store_name}.pkl"):
         with open(f"{store_name}.pkl", "rb") as f:
             VectorStore = pickle.load(f)
     else:
-        embeddings = OpenAIEmbeddings()  # No api_key parameter here
+        embeddings = OpenAIEmbeddings()
         VectorStore = FAISS.from_texts(chunks, embedding=embeddings)
         with open(f"{store_name}.pkl", "wb") as f:
             pickle.dump(VectorStore, f)
 
     return VectorStore
 
-
 def load_chatbot(max_tokens=120):
     return load_qa_chain(llm=OpenAI(temperature=0.5, max_tokens=max_tokens), chain_type="stuff") 
 
-
 def display_chat_history(chat_history):
     for chat in chat_history:
         background_color = "#FFA07A" if chat[2] == "new" else "#acf" if chat[0] == "User" else "#caf"
         st.markdown(f"<div style='background-color: {background_color}; padding: 10px; border-radius: 10px; margin: 10px;'>{chat[0]}: {chat[1]}</div>", unsafe_allow_html=True)
 
+def remove_incomplete_sentences(text):
+    sentences = sent_tokenize(text)
+    complete_sentences = [sent for sent in sentences if sent.endswith(('.', '!', '?'))]
+    return ' '.join(complete_sentences)
+
+def remove_redundant_information(text):
+    sentences = sent_tokenize(text)
+    unique_sentences = list(set(sentences))
+    return ' '.join(unique_sentences)
+
+# Define a maximum token limit to avoid infinite loops
+MAX_TOKEN_LIMIT = 400
+
 def main():
     st.title("BinDocs Chat App")
 
@@ -91,36 +98,33 @@ def main():
         query = st.text_input("Ask questions about your PDF file (in any preferred language):")
 
         if st.button("Ask") or (query and query != st.session_state.get('last_input', '')):
-            st.session_state['last_input'] = query  # Save the current query as the last input
+            st.session_state['last_input'] = query
             st.session_state['chat_history'].append(("User", query, "new"))
 
             loading_message = st.empty()
             loading_message.text('Bot is thinking...')
             
             VectorStore = load_pdf(pdf)
-            max_tokens = 100  # Initial max tokens
+            max_tokens = 100
             chain = load_chatbot(max_tokens=max_tokens)
-            docs = VectorStore.similarity_search(query=query, k=1)
+            docs = VectorStore.similarity_search(query=query, k=2)
+            
             with get_openai_callback() as cb:
                 response = chain.run(input_documents=docs, question=query)
-
             
-            # Filtering similar responses (a simple example using set to remove duplicate sentences)
-            response_sentences = response.split('. ')
-            unique_sentences = set(response_sentences)
-            filtered_response = '. '.join(unique_sentences)
-                
-            # Check if the filtered response ends with a sentence-ending punctuation
+            # Post-processing to remove incomplete sentences and redundant information
+            filtered_response = remove_incomplete_sentences(response)
+            filtered_response = remove_redundant_information(filtered_response)
+            
+            # Check if the response ends with a sentence-ending punctuation
             while not filtered_response.strip().endswith(('.', '!', '?')) and max_tokens < MAX_TOKEN_LIMIT:
                 max_tokens += 50  # Increase the max_tokens limit
                 chain = load_chatbot(max_tokens=max_tokens)
                 additional_response = chain.run(input_documents=docs, question=query)
-                filtered_response += additional_response  # Append the additional response to the filtered response
+                filtered_response += additional_response  # Append the additional response to the filtered_response
 
             st.session_state['chat_history'].append(("Bot", filtered_response, "new"))
 
-
-
             # Display new messages at the bottom
             new_messages = st.session_state['chat_history'][-2:]
             for chat in new_messages:
@@ -138,8 +142,6 @@ def main():
         # Mark all messages as old after displaying
         st.session_state['chat_history'] = [(sender, msg, "old") for sender, msg, _ in st.session_state['chat_history']]
 
-# Define a maximum token limit to avoid infinite loops
-MAX_TOKEN_LIMIT = 400
-
 if __name__ == "__main__":
     main()
+