update langchain

app.py

@@ -39,6 +39,9 @@ from langchain.memory import ConversationBufferMemory
 
 from pydantic.v1 import BaseModel, Field
 
+# Import the separate file that contains our list of URLs
+from url_list import URLS
+
 
 """
 =========================================================
@@ -54,7 +57,7 @@ class_labels = ['cardboard', 'glass', 'metal', 'paper', 'plastic', 'trash']
 def predict_image(input_image):
     """
     Resize the user-uploaded image and preprocess it so that it can be fed
-    into the EfficientNetB0 model. The model then returns a dictionary of 
+    into the EfficientNetB0 model. The model then returns a dictionary of
     class probabilities.
     """
     # Resize the image (note the target dimensions)
@@ -84,7 +87,6 @@ image_gradio_app = gr.Interface(
     theme=theme
 )
 
-
 """
 =========================================================
  3) CHATBOT MODEL SETUP
@@ -94,86 +96,6 @@ image_gradio_app = gr.Interface(
 user_agent = UserAgent().random
 header_template = {"User-Agent": user_agent}
 
-# 3.2) List of URLs to load for retrieval
-URLS = [
-    "https://www.epa.gov/recycle/frequent-questions-recycling",
-    "https://www.whitehorsedc.gov.uk/vale-of-white-horse-district-council/recycling-rubbish-and-waste/lets-get-real-about-recycling/",
-    "https://www.teimas.com/blog/13-preguntas-y-respuestas-sobre-la-ley-de-residuos-07-2022",
-    "https://www.molok.com/es/blog/gestion-de-residuos-solidos-urbanos-rsu-10-dudas-comunes",
-    "https://espanol.epa.gov/espanol/el-reciclaje#valelapena",
-    "https://espanol.epa.gov/espanol/preguntas-frecuentes-sobre-reciclado-de-plastico-y-elaboracion-de-abono-vegetal",
-    "https://espanol.epa.gov/espanol/consejo-del-dia-como-reciclo-mis",
-    "https://espanol.epa.gov/espanol/recursos-para-reciclar-dispositivos-electronicos",
-    "https://www.epa.gov/recycle/electronics-donation-and-recycling",
-    "https://reducereutilizarecicla.org/que-es-el-reciclaje/",
-    "https://reducereutilizarecicla.org/contenedores-de-reciclaje/",
-    "https://reducereutilizarecicla.org/contenedores-de-reciclaje/contenedor-amarillo/",
-    "https://reducereutilizarecicla.org/contenedores-de-reciclaje/contenedor-azul/",
-    "https://reducereutilizarecicla.org/contenedores-de-reciclaje/contenedor-verde/",
-    "https://reducereutilizarecicla.org/contenedores-de-reciclaje/contenedor-marron-organico/",
-    "https://reducereutilizarecicla.org/contenedores-de-reciclaje/contenedor-gris-restos/",
-    "https://reducereutilizarecicla.org/contenedores-de-reciclaje/punto-limpio/",
-    "https://reducereutilizarecicla.org/donde-tirar-auriculares/",
-    "https://reducereutilizarecicla.org/donde-tirar-sartenes/",
-    "https://reducereutilizarecicla.org/donde-tirar-aceite-usado/",
-    "https://reducereutilizarecicla.org/como-se-reciclan-los-envases-tipo-brik/",
-    "https://reducereutilizarecicla.org/los-envases-del-verano/",
-    "https://reducereutilizarecicla.org/donde-tirar-radiografias/",
-    "https://reducereutilizarecicla.org/envases-ecologicos/",
-    "https://reducereutilizarecicla.org/donde-tirar-los-restos-de-pintura/",
-    "https://reducereutilizarecicla.org/valorizacion-de-residuos/",
-    "https://reducereutilizarecicla.org/como-reciclar-pilas/",
-    "https://reducereutilizarecicla.org/como-reciclar-capsulas-de-cafe/",
-    "https://reducereutilizarecicla.org/reciclando-cd/",
-    "https://reducereutilizarecicla.org/donde-tirar-neumaticos/",
-    "https://reducereutilizarecicla.org/como-reciclar-una-canasta-de-mimbre/",
-    "https://reducereutilizarecicla.org/como-funciona-el-contenedor-amarillo/",
-    "https://reducereutilizarecicla.org/donde-se-tiran-los-vapers/",
-    "https://reducereutilizarecicla.org/cuanto-tarda-una-bolsa-biodegradable-en-degradarse/",
-    "https://reducereutilizarecicla.org/donde-se-reciclan-los-juguetes/",
-    "https://reducereutilizarecicla.org/objetos-que-se-pueden-reutilizar/",
-    "https://reducereutilizarecicla.org/la-parafina-se-puede-reutilizar/",
-    "https://reducereutilizarecicla.org/planta-de-reciclaje-de-papel/",
-    "https://reducereutilizarecicla.org/como-saber-si-un-envase-es-reciclable/",
-    "https://reducereutilizarecicla.org/reutilizar-vasos-de-vela/",
-    "https://reducereutilizarecicla.org/bolsas-frio-calor/",
-    "https://reducereutilizarecicla.org/reciclar-y-reutilizar-materiales-de-construccion/",
-    "https://reducereutilizarecicla.org/que-es-exactamente-el-pet/",
-    "https://reducereutilizarecicla.org/tipos-de-reciclaje/",
-    "https://reducereutilizarecicla.org/que-hacer-con-palets-reciclados/",
-    "https://reducereutilizarecicla.org/vertederos-controlados/",
-    "https://reducereutilizarecicla.org/donde-tirar-escombros/",
-    "https://reducereutilizarecicla.org/como-reciclar-los-residuos-de-ps-poliestireno/",
-    "https://reducereutilizarecicla.org/tirar-la-basura-sin-bolsas/",
-    "https://reducereutilizarecicla.org/tirar-el-palo-de-la-fregona/",
-    "https://reducereutilizarecicla.org/la-mejor-manera-de-reciclar-una-pala-de-padel/",
-    "https://reducereutilizarecicla.org/sabes-donde-tirar-las-llantas-viejas-de-un-coche/",
-    "https://reducereutilizarecicla.org/sabes-donde-tirar-el-arbol-de-navidad/",
-    "https://reducereutilizarecicla.org/clavos-tornillos-herramientas-donde-tirar-hierro/",
-    "https://reducereutilizarecicla.org/donde-tirar-un-secador-de-pelo-contenedor-o-punto-limpio/",
-    "https://reducereutilizarecicla.org/donde-tirar-electrodomesticos/",
-    "https://reducereutilizarecicla.org/donde-puedo-tirar-ramas-de-arboles/",
-    "https://reducereutilizarecicla.org/donde-tirar-escombros/",
-    "https://reducereutilizarecicla.org/donde-se-tira-el-muerdago-quemado/",
-    "https://reducereutilizarecicla.org/sandalias-caucho-reciclado-neumaticos/",
-    "https://reducereutilizarecicla.org/ideas-para-reciclar-aspas-de-ventilador-de-techo/",
-    "https://reducereutilizarecicla.org/reciclar-sacos-dormir/",
-    "https://reducereutilizarecicla.org/reciclar-sillas-playa/",
-    "https://reducereutilizarecicla.org/donde-tirar-antipolillas/",
-    "https://reducereutilizarecicla.org/que-hacer-con-los-juguetes-viejos/",
-    "https://reducereutilizarecicla.org/como-utilizar-las-mascarillas-y-el-gel-hidroalcoholico-en-la-playa/",
-    "https://reducereutilizarecicla.org/ideas-para-reciclar-un-ventilador-de-pie/",
-    "https://reducereutilizarecicla.org/donde-tirar-gasoil/",
-    "https://reducereutilizarecicla.org/donde-puedo-tirar-basura-electronica/",
-    "https://reducereutilizarecicla.org/donde-tirar-agujas/",
-    "https://reducereutilizarecicla.org/donde-tirar-residuos-peligrosos/",
-    "https://reducereutilizarecicla.org/donde-tirar-los-cables/",
-    "https://reducereutilizarecicla.org/donde-tirar-bicicletas/",
-    "https://reducereutilizarecicla.org/donde-tirar-maletas/",
-    "https://reducereutilizarecicla.org/como-reciclar-una-pantalla/",
-    "https://reducereutilizarecicla.org/donde-tirar-ropa-usada/"
-]
-
 
 @tenacity.retry(
     wait=tenacity.wait_fixed(3),   # wait 3 seconds between retries
@@ -182,7 +104,7 @@ URLS = [
 )
 def load_url(url):
     """
-    Use the WebBaseLoader for a single URL. 
+    Use the WebBaseLoader for a single URL.
     The function is retried if it fails due to connection issues.
     """
     loader = WebBaseLoader(
@@ -194,7 +116,7 @@ def load_url(url):
 
 def safe_load_all_urls(urls):
     """
-    Safely load documents from a list of URLs. 
+    Safely load documents from a list of URLs.
     Any URL that fails after the specified number of retries is skipped.
     """
     all_docs = []
@@ -207,11 +129,10 @@ def safe_load_all_urls(urls):
             print(f"Skipping URL due to error: {link}\nError: {e}\n")
     return all_docs
 
-
-# 3.3) Actually load the data from all URLs
+# 3.2) Actually load the data from all URLs (imported from url_list.py)
 all_loaded_docs = safe_load_all_urls(URLS)
 
-# 3.4) Split the documents into manageable chunks
+# 3.3) Split the documents into manageable chunks
 text_splitter = RecursiveCharacterTextSplitter(
     chunk_size=1024,
     chunk_overlap=150,
@@ -219,27 +140,26 @@ text_splitter = RecursiveCharacterTextSplitter(
 )
 docs = text_splitter.split_documents(all_loaded_docs)
 
-# 3.5) Create embeddings
+# 3.4) Create embeddings
 embeddings = HuggingFaceEmbeddings(model_name='thenlper/gte-small')
 
-# 3.6) Create a persistent directory to store vector DB
+# 3.5) Create a persistent directory to store vector DB
 persist_directory = 'docs/chroma/'
 shutil.rmtree(persist_directory, ignore_errors=True)  # remove old DB files
 
-# 3.7) Build Chroma vector store
+# 3.6) Build Chroma vector store
 vectordb = Chroma.from_documents(
     documents=docs,
     embedding=embeddings,
     persist_directory=persist_directory
 )
 
-# 3.8) Create a retriever 
+# 3.7) Create a retriever
 retriever = vectordb.as_retriever(
     search_kwargs={"k": 2},
     search_type="mmr"
 )
 
-
 """
 =========================================================
  4) PROMPT & CHAIN SETUP
@@ -254,8 +174,8 @@ parser = PydanticOutputParser(pydantic_object=FinalAnswer)
 
 # 4.2) Prompt template: system instructions
 template = """
-Your name is Greta and you are a recycling chatbot with the objective to anwer questions from user in English or Spanish /
-Has sido diseñado y creado por el Grupo 1 del Máster en Data Science & Big Data de la promoción 2023/2024 de la Universidad Complutense de Madrid. Este grupo está fromado por Rocío, María Guillermo, Alejandra, Paloma y Álvaro /
+Your name is Greta and you are a recycling chatbot with the objective to answer questions from user in English or Spanish /
+Has sido diseñado y creado por el Grupo 1 del Máster en Data Science & Big Data de la promoción 2023/2024 de la Universidad Complutense de Madrid. Este grupo está formado por Rocío, María Guillermo, Alejandra, Paloma y Álvaro /
 Use the following pieces of context to answer the question /
 If the question is English answer in English /
 If the question is Spanish answer in Spanish /
@@ -304,7 +224,6 @@ qa_chain = ConversationalRetrievalChain.from_llm(
     output_key='output'
 )
 
-
 def chat_interface(question, history):
     """
     This function processes the user's question through the qa_chain,
@@ -330,7 +249,6 @@ chatbot_gradio_app = gr.ChatInterface(
     title="<span style='color: rgb(243, 239, 224);'>Green Greta</span>"
 )
 
-
 """
 =========================================================
  5) BANNER / WELCOME TAB
@@ -359,7 +277,6 @@ banner_tab_content = """
 """
 banner_tab = gr.Markdown(banner_tab_content)
 
-
 """
 =========================================================
  6) GRADIO FINAL APP: TABS

url_list.py

@@ -0,0 +1,84 @@
+# url_list.py
+
+"""
+This file contains the list of URLs to be loaded by the main app.py file.
+"""
+
+URLS = [
+    "https://www.epa.gov/recycle/frequent-questions-recycling",
+    "https://www.whitehorsedc.gov.uk/vale-of-white-horse-district-council/recycling-rubbish-and-waste/lets-get-real-about-recycling/",
+    "https://www.teimas.com/blog/13-preguntas-y-respuestas-sobre-la-ley-de-residuos-07-2022",
+    "https://www.molok.com/es/blog/gestion-de-residuos-solidos-urbanos-rsu-10-dudas-comunes",
+    "https://espanol.epa.gov/espanol/el-reciclaje#valelapena",
+    "https://espanol.epa.gov/espanol/preguntas-frecuentes-sobre-reciclado-de-plastico-y-elaboracion-de-abono-vegetal",
+    "https://espanol.epa.gov/espanol/consejo-del-dia-como-reciclo-mis",
+    "https://espanol.epa.gov/espanol/recursos-para-reciclar-dispositivos-electronicos",
+    "https://www.epa.gov/recycle/electronics-donation-and-recycling",
+    "https://reducereutilizarecicla.org/que-es-el-reciclaje/",
+    "https://reducereutilizarecicla.org/contenedores-de-reciclaje/",
+    "https://reducereutilizarecicla.org/contenedores-de-reciclaje/contenedor-amarillo/",
+    "https://reducereutilizarecicla.org/contenedores-de-reciclaje/contenedor-azul/",
+    "https://reducereutilizarecicla.org/contenedores-de-reciclaje/contenedor-verde/",
+    "https://reducereutilizarecicla.org/contenedores-de-reciclaje/contenedor-marron-organico/",
+    "https://reducereutilizarecicla.org/contenedores-de-reciclaje/contenedor-gris-restos/",
+    "https://reducereutilizarecicla.org/contenedores-de-reciclaje/punto-limpio/",
+    "https://reducereutilizarecicla.org/donde-tirar-auriculares/",
+    "https://reducereutilizarecicla.org/donde-tirar-sartenes/",
+    "https://reducereutilizarecicla.org/donde-tirar-aceite-usado/",
+    "https://reducereutilizarecicla.org/como-se-reciclan-los-envases-tipo-brik/",
+    "https://reducereutilizarecicla.org/los-envases-del-verano/",
+    "https://reducereutilizarecicla.org/donde-tirar-radiografias/",
+    "https://reducereutilizarecicla.org/envases-ecologicos/",
+    "https://reducereutilizarecicla.org/donde-tirar-los-restos-de-pintura/",
+    "https://reducereutilizarecicla.org/valorizacion-de-residuos/",
+    "https://reducereutilizarecicla.org/como-reciclar-pilas/",
+    "https://reducereutilizarecicla.org/como-reciclar-capsulas-de-cafe/",
+    "https://reducereutilizarecicla.org/reciclando-cd/",
+    "https://reducereutilizarecicla.org/donde-tirar-neumaticos/",
+    "https://reducereutilizarecicla.org/como-reciclar-una-canasta-de-mimbre/",
+    "https://reducereutilizarecicla.org/como-funciona-el-contenedor-amarillo/",
+    "https://reducereutilizarecicla.org/donde-se-tiran-los-vapers/",
+    "https://reducereutilizarecicla.org/cuanto-tarda-una-bolsa-biodegradable-en-degradarse/",
+    "https://reducereutilizarecicla.org/donde-se-reciclan-los-juguetes/",
+    "https://reducereutilizarecicla.org/objetos-que-se-pueden-reutilizar/",
+    "https://reducereutilizarecicla.org/la-parafina-se-puede-reutilizar/",
+    "https://reducereutilizarecicla.org/planta-de-reciclaje-de-papel/",
+    "https://reducereutilizarecicla.org/como-saber-si-un-envase-es-reciclable/",
+    "https://reducereutilizarecicla.org/reutilizar-vasos-de-vela/",
+    "https://reducereutilizarecicla.org/bolsas-frio-calor/",
+    "https://reducereutilizarecicla.org/reciclar-y-reutilizar-materiales-de-construccion/",
+    "https://reducereutilizarecicla.org/que-es-exactamente-el-pet/",
+    "https://reducereutilizarecicla.org/tipos-de-reciclaje/",
+    "https://reducereutilizarecicla.org/que-hacer-con-palets-reciclados/",
+    "https://reducereutilizarecicla.org/vertederos-controlados/",
+    "https://reducereutilizarecicla.org/donde-tirar-escombros/",
+    "https://reducereutilizarecicla.org/como-reciclar-los-residuos-de-ps-poliestireno/",
+    "https://reducereutilizarecicla.org/tirar-la-basura-sin-bolsas/",
+    "https://reducereutilizarecicla.org/tirar-el-palo-de-la-fregona/",
+    "https://reducereutilizarecicla.org/la-mejor-manera-de-reciclar-una-pala-de-padel/",
+    "https://reducereutilizarecicla.org/sabes-donde-tirar-las-llantas-viejas-de-un-coche/",
+    "https://reducereutilizarecicla.org/sabes-donde-tirar-el-arbol-de-navidad/",
+    "https://reducereutilizarecicla.org/clavos-tornillos-herramientas-donde-tirar-hierro/",
+    "https://reducereutilizarecicla.org/donde-tirar-un-secador-de-pelo-contenedor-o-punto-limpio/",
+    "https://reducereutilizarecicla.org/donde-tirar-electrodomesticos/",
+    "https://reducereutilizarecicla.org/donde-puedo-tirar-ramas-de-arboles/",
+    "https://reducereutilizarecicla.org/donde-tirar-escombros/",
+    "https://reducereutilizarecicla.org/donde-se-tira-el-muerdago-quemado/",
+    "https://reducereutilizarecicla.org/sandalias-caucho-reciclado-neumaticos/",
+    "https://reducereutilizarecicla.org/ideas-para-reciclar-aspas-de-ventilador-de-techo/",
+    "https://reducereutilizarecicla.org/reciclar-sacos-dormir/",
+    "https://reducereutilizarecicla.org/reciclar-sillas-playa/",
+    "https://reducereutilizarecicla.org/donde-tirar-antipolillas/",
+    "https://reducereutilizarecicla.org/que-hacer-con-los-juguetes-viejos/",
+    "https://reducereutilizarecicla.org/como-utilizar-las-mascarillas-y-el-gel-hidroalcoholico-en-la-playa/",
+    "https://reducereutilizarecicla.org/ideas-para-reciclar-un-ventilador-de-pie/",
+    "https://reducereutilizarecicla.org/donde-tirar-gasoil/",
+    "https://reducereutilizarecicla.org/donde-puedo-tirar-basura-electronica/",
+    "https://reducereutilizarecicla.org/donde-tirar-agujas/",
+    "https://reducereutilizarecicla.org/donde-tirar-residuos-peligrosos/",
+    "https://reducereutilizarecicla.org/donde-tirar-los-cables/",
+    "https://reducereutilizarecicla.org/donde-tirar-bicicletas/",
+    "https://reducereutilizarecicla.org/donde-tirar-maletas/",
+    "https://reducereutilizarecicla.org/como-reciclar-una-pantalla/",
+    "https://reducereutilizarecicla.org/donde-tirar-ropa-usada/"
+]