user agent

app.py

@@ -33,6 +33,7 @@ from langchain.output_parsers import PydanticOutputParser
 from langchain_community.llms import HuggingFaceHub
 from langchain_community.document_loaders import WebBaseLoader
 
+from fake_useragent import UserAgent
 
 custom_title = "<span style='color: rgb(243, 239, 224);'>Green Greta</span>"
 
@@ -72,8 +73,96 @@ image_gradio_app = gr.Interface(
 
 
 # Cell 2: ChatBot Model
-loader = WebBaseLoader(["https://www.epa.gov/recycle/frequent-questions-recycling", "https://www.whitehorsedc.gov.uk/vale-of-white-horse-district-council/recycling-rubbish-and-waste/lets-get-real-about-recycling/", "https://www.teimas.com/blog/13-preguntas-y-respuestas-sobre-la-ley-de-residuos-07-2022", "https://www.molok.com/es/blog/gestion-de-residuos-solidos-urbanos-rsu-10-dudas-comunes", "https://espanol.epa.gov/espanol/el-reciclaje#valelapena","https://espanol.epa.gov/espanol/preguntas-frecuentes-sobre-reciclado-de-plastico-y-elaboracion-de-abono-vegetal","https://espanol.epa.gov/espanol/consejo-del-dia-como-reciclo-mis","https://espanol.epa.gov/espanol/recursos-para-reciclar-dispositivos-electronicos","https://www.epa.gov/recycle/electronics-donation-and-recycling","https://reducereutilizarecicla.org/que-es-el-reciclaje/","https://reducereutilizarecicla.org/contenedores-de-reciclaje/", "https://reducereutilizarecicla.org/contenedores-de-reciclaje/contenedor-amarillo/","https://reducereutilizarecicla.org/contenedores-de-reciclaje/contenedor-azul/","https://reducereutilizarecicla.org/contenedores-de-reciclaje/contenedor-verde/","https://reducereutilizarecicla.org/contenedores-de-reciclaje/contenedor-marron-organico/","https://reducereutilizarecicla.org/contenedores-de-reciclaje/contenedor-gris-restos/","https://reducereutilizarecicla.org/contenedores-de-reciclaje/punto-limpio/", "https://reducereutilizarecicla.org/donde-tirar-auriculares/","https://reducereutilizarecicla.org/donde-tirar-sartenes/","https://reducereutilizarecicla.org/donde-tirar-aceite-usado/","https://reducereutilizarecicla.org/como-se-reciclan-los-envases-tipo-brik/","https://reducereutilizarecicla.org/los-envases-del-verano/", "https://reducereutilizarecicla.org/donde-tirar-radiografias/","https://reducereutilizarecicla.org/envases-ecologicos/","https://reducereutilizarecicla.org/donde-tirar-los-restos-de-pintura/","https://reducereutilizarecicla.org/valorizacion-de-residuos/","https://reducereutilizarecicla.org/como-reciclar-pilas/","https://reducereutilizarecicla.org/como-reciclar-capsulas-de-cafe/","https://reducereutilizarecicla.org/reciclando-cd/", "https://reducereutilizarecicla.org/donde-tirar-neumaticos/","https://reducereutilizarecicla.org/como-reciclar-una-canasta-de-mimbre/","https://reducereutilizarecicla.org/como-funciona-el-contenedor-amarillo/", "https://reducereutilizarecicla.org/donde-se-tiran-los-vapers/","https://reducereutilizarecicla.org/cuanto-tarda-una-bolsa-biodegradable-en-degradarse/", "https://reducereutilizarecicla.org/donde-se-reciclan-los-juguetes/","https://reducereutilizarecicla.org/objetos-que-se-pueden-reutilizar/","https://reducereutilizarecicla.org/la-parafina-se-puede-reutilizar/","https://reducereutilizarecicla.org/planta-de-reciclaje-de-papel/", "https://reducereutilizarecicla.org/como-saber-si-un-envase-es-reciclable/", "https://reducereutilizarecicla.org/reutilizar-vasos-de-vela/", "https://reducereutilizarecicla.org/bolsas-frio-calor/", "https://reducereutilizarecicla.org/reciclar-y-reutilizar-materiales-de-construccion/", "https://reducereutilizarecicla.org/que-es-exactamente-el-pet/", "https://reducereutilizarecicla.org/tipos-de-reciclaje/", "https://reducereutilizarecicla.org/que-hacer-con-palets-reciclados/", "https://reducereutilizarecicla.org/vertederos-controlados/", "https://reducereutilizarecicla.org/donde-tirar-escombros/","https://reducereutilizarecicla.org/como-reciclar-los-residuos-de-ps-poliestireno/","https://reducereutilizarecicla.org/tirar-la-basura-sin-bolsas/","https://reducereutilizarecicla.org/tirar-el-palo-de-la-fregona/","https://reducereutilizarecicla.org/la-mejor-manera-de-reciclar-una-pala-de-padel/", "https://reducereutilizarecicla.org/sabes-donde-tirar-las-llantas-viejas-de-un-coche/","https://reducereutilizarecicla.org/sabes-donde-tirar-el-arbol-de-navidad/","https://reducereutilizarecicla.org/clavos-tornillos-herramientas-donde-tirar-hierro/","https://reducereutilizarecicla.org/donde-tirar-un-secador-de-pelo-contenedor-o-punto-limpio/","https://reducereutilizarecicla.org/donde-tirar-electrodomesticos/","https://reducereutilizarecicla.org/donde-puedo-tirar-ramas-de-arboles/", "https://reducereutilizarecicla.org/donde-tirar-escombros/","https://reducereutilizarecicla.org/donde-se-tira-el-muerdago-quemado/","https://reducereutilizarecicla.org/sandalias-caucho-reciclado-neumaticos/","https://reducereutilizarecicla.org/ideas-para-reciclar-aspas-de-ventilador-de-techo/","https://reducereutilizarecicla.org/reciclar-sacos-dormir/","https://reducereutilizarecicla.org/reciclar-sillas-playa/","https://reducereutilizarecicla.org/donde-tirar-antipolillas/","https://reducereutilizarecicla.org/que-hacer-con-los-juguetes-viejos/","https://reducereutilizarecicla.org/como-utilizar-las-mascarillas-y-el-gel-hidroalcoholico-en-la-playa/","https://reducereutilizarecicla.org/ideas-para-reciclar-un-ventilador-de-pie/","https://reducereutilizarecicla.org/donde-tirar-gasoil/","https://reducereutilizarecicla.org/donde-puedo-tirar-basura-electronica/","https://reducereutilizarecicla.org/donde-tirar-agujas/", "https://reducereutilizarecicla.org/donde-tirar-residuos-peligrosos/", "https://reducereutilizarecicla.org/donde-tirar-los-cables/", "https://reducereutilizarecicla.org/donde-tirar-bicicletas/", "https://reducereutilizarecicla.org/donde-tirar-maletas/", "https://reducereutilizarecicla.org/como-reciclar-una-pantalla/", "https://reducereutilizarecicla.org/metales-reciclables/","https://reducereutilizarecicla.org/donde-tirar-caja-de-helado/", "https://reducereutilizarecicla.org/como-reciclar-perchas-de-plastico/","https://reducereutilizarecicla.org/donde-tirar-un-jarron-de-ceramica/","https://reducereutilizarecicla.org/donde-tirar-sanitarios/", "https://reducereutilizarecicla.org/reciclar-bombonas-de-camping-gas/", "https://reducereutilizarecicla.org/donde-tirar-aceite-usado-de-motor/", "https://reducereutilizarecicla.org/como-reciclar-rotuladores-subrayadores-y-boligrafos/", "https://reducereutilizarecicla.org/donde-tirar-un-ordenador/", "https://reducereutilizarecicla.org/donde-tirar-un-termometro-de-mercurio/", "https://reducereutilizarecicla.org/tirar-nevera-vieja/","https://reducereutilizarecicla.org/que-cosas-pueden-ser-recicladas/","https://reducereutilizarecicla.org/donde-tirar-los-pintaunas/","https://reducereutilizarecicla.org/donde-tirar-bombona-de-helio/", "https://reducereutilizarecicla.org/donde-tirar-alfombras/", "https://reducereutilizarecicla.org/donde-tirar-impresoras-y-sus-cartuchos-o-toner/", "https://reducereutilizarecicla.org/donde-tirar-aguarras/","https://reducereutilizarecicla.org/donde-tirar-discos-duros/","https://reducereutilizarecicla.org/donde-tirar-azulejos/","https://reducereutilizarecicla.org/donde-tirar-diapositivas/","https://reducereutilizarecicla.org/donde-tirar-jeringuillas-usadas/","https://reducereutilizarecicla.org/donde-tirar-cintas-vhs/","https://reducereutilizarecicla.org/donde-tirar-gomaespuma/", "https://reducereutilizarecicla.org/donde-tirar-los-botes-de-pintura/", "https://reducereutilizarecicla.org/donde-se-recicla-la-madera/", "https://reducereutilizarecicla.org/donde-tirar-discos-de-vinilo/", "https://reducereutilizarecicla.org/donde-tirar-imanes/", "https://reducereutilizarecicla.org/donde-tirar-fluorescentes/", "https://reducereutilizarecicla.org/donde-tirar-un-microondas/", "https://reducereutilizarecicla.org/reciclar-toallas/", "https://reducereutilizarecicla.org/reciclar-vaqueros/","https://reducereutilizarecicla.org/como-se-recicla-la-tela/", "https://reducereutilizarecicla.org/contenedor-rojo-ropa/", "https://reducereutilizarecicla.org/reciclar-chanclas/","https://reducereutilizarecicla.org/reciclar-banadores/","https://reducereutilizarecicla.org/asi-funciona-el-reciclaje-de-cremalleras/","https://reducereutilizarecicla.org/donde-tirar-zapatos/","https://reducereutilizarecicla.org/como-reciclar-una-camisa/","https://reducereutilizarecicla.org/donde-tirar-un-mantel-de-tela-sucio/","https://reducereutilizarecicla.org/contenedores-de-ropa/","https://reducereutilizarecicla.org/que-cosas-pueden-ser-recicladas/","https://reducereutilizarecicla.org/los-textiles-se-vuelven-ecologicos/","https://reducereutilizarecicla.org/donde-tirar-ropa-vieja/","https://espanol.epa.gov/espanol/terminos-0-9","https://espanol.epa.gov/espanol/terminos","https://espanol.epa.gov/espanol/terminos-b","https://espanol.epa.gov/espanol/terminos-c","https://espanol.epa.gov/espanol/terminos-d","https://espanol.epa.gov/espanol/terminos-e","https://espanol.epa.gov/espanol/terminos-f","https://espanol.epa.gov/espanol/terminos-g","https://espanol.epa.gov/espanol/terminos-h","https://espanol.epa.gov/espanol/terminos-i","https://espanol.epa.gov/espanol/terminos-j","https://espanol.epa.gov/espanol/terminos-l","https://espanol.epa.gov/espanol/terminos-m","https://espanol.epa.gov/espanol/terminos-n","https://espanol.epa.gov/espanol/terminos-o","https://espanol.epa.gov/espanol/terminos-p","https://espanol.epa.gov/espanol/terminos-q","https://espanol.epa.gov/espanol/terminos-r","https://espanol.epa.gov/espanol/terminos-s","https://espanol.epa.gov/espanol/terminos-t","https://espanol.epa.gov/espanol/terminos-u","https://espanol.epa.gov/espanol/terminos-v","https://espanol.epa.gov/espanol/terminos-w-x-y-z#W", "https://espanol.epa.gov/espanol/la-importancia-de-la-educacion-ambiental","https://espanol.epa.gov/espanol/consejo-del-dia-que-puede-hacer-para-reciclar-por-estacion","https://espanol.epa.gov/espanol/lo-que-puede-hacer-usted-acerca-de-la-contaminacion-por-basura-0","https://espanol.epa.gov/espanol/diez-maneras-de-eliminar-el-embalaje-y-las-envolturas-de-su-vida","https://blog.cerdanyaecoresort.com/sostenibilidad-ambiental-que-es-y-como-mejorarla-en-casa/","https://www.imh.eus/es/imh/comunicacion/docu-libre/reduccion-residuos","https://archivo-es.greenpeace.org/espana/Global/espana/report/contaminacion/Guia%20Transform-accion%20residuos.pdf","https://elpais.com/escaparate/2023-01-10/11-productos-sostenibles-de-uso-cotidiano-para-reducir-el-consumo-de-plastico-y-generar-menos-residuos.html", "https://elpais.com/escaparate/2023-01-10/11-productos-sostenibles-de-uso-cotidiano-para-reducir-el-consumo-de-plastico-y-generar-menos-residuos.html#", "https://www.iberdrola.com/sostenibilidad/productos-ecologicos", "https://www.iberdrola.com/sostenibilidad/reciclaje-para-ninos","https://www.miteco.gob.es/es/ceneam/recursos/pag-web/programas-planes/voluntariado-ong-internacionales.html", "https://www.iberdrola.com/compromiso-social/voluntariado-corporativo", "https://reducereutilizarecicla.org/reciclos-ecoembes/","https://reducereutilizarecicla.org/goma-eva-es-reciclable/", "https://reducereutilizarecicla.org/se-puede-generar-energia-en-los-vertederos/", "https://reducereutilizarecicla.org/tirar-la-basura-sin-bolsas/","https://reducereutilizarecicla.org/por-que-es-importante-saber-como-reutilizar-camisetas/","https://reducereutilizarecicla.org/reutilizar-pantalones/","https://reducereutilizarecicla.org/poliester-reciclado/","https://reducereutilizarecicla.org/ropa-con-materiales-reciclados/", "https://reducereutilizarecicla.org/contenedores-caseros-para-reciclar/","https://www.miteco.gob.es/es/calidad-y-evaluacion-ambiental/temas/prevencion-y-gestion-residuos/flujos/domesticos/gestion/sistema-recogida/puntos-limpios.html","https://punto-limpio.info/","https://reducereutilizarecicla.org/ecopuntos-que-son-y-donde-estan/"])
+
+# Generate a random user agent
+user_agent = UserAgent().random
+header_template = {"User-Agent": user_agent}
+
+# Create the loader with the headers
+loader = WebBaseLoader(
+    web_paths=[
+        "https://www.epa.gov/recycle/frequent-questions-recycling",
+        "https://www.whitehorsedc.gov.uk/vale-of-white-horse-district-council/recycling-rubbish-and-waste/lets-get-real-about-recycling/",
+        "https://www.teimas.com/blog/13-preguntas-y-respuestas-sobre-la-ley-de-residuos-07-2022",
+        "https://www.molok.com/es/blog/gestion-de-residuos-solidos-urbanos-rsu-10-dudas-comunes",
+        "https://espanol.epa.gov/espanol/el-reciclaje#valelapena",
+        "https://espanol.epa.gov/espanol/preguntas-frecuentes-sobre-reciclado-de-plastico-y-elaboracion-de-abono-vegetal",
+        "https://espanol.epa.gov/espanol/consejo-del-dia-como-reciclo-mis",
+        "https://espanol.epa.gov/espanol/recursos-para-reciclar-dispositivos-electronicos",
+        "https://www.epa.gov/recycle/electronics-donation-and-recycling",
+        "https://reducereutilizarecicla.org/que-es-el-reciclaje/",
+        "https://reducereutilizarecicla.org/contenedores-de-reciclaje/",
+        "https://reducereutilizarecicla.org/contenedores-de-reciclaje/contenedor-amarillo/",
+        "https://reducereutilizarecicla.org/contenedores-de-reciclaje/contenedor-azul/",
+        "https://reducereutilizarecicla.org/contenedores-de-reciclaje/contenedor-verde/",
+        "https://reducereutilizarecicla.org/contenedores-de-reciclaje/contenedor-marron-organico/",
+        "https://reducereutilizarecicla.org/contenedores-de-reciclaje/contenedor-gris-restos/",
+        "https://reducereutilizarecicla.org/contenedores-de-reciclaje/punto-limpio/",
+        "https://reducereutilizarecicla.org/donde-tirar-auriculares/",
+        "https://reducereutilizarecicla.org/donde-tirar-sartenes/",
+        "https://reducereutilizarecicla.org/donde-tirar-aceite-usado/",
+        "https://reducereutilizarecicla.org/como-se-reciclan-los-envases-tipo-brik/",
+        "https://reducereutilizarecicla.org/los-envases-del-verano/",
+        "https://reducereutilizarecicla.org/donde-tirar-radiografias/",
+        "https://reducereutilizarecicla.org/envases-ecologicos/",
+        "https://reducereutilizarecicla.org/donde-tirar-los-restos-de-pintura/",
+        "https://reducereutilizarecicla.org/valorizacion-de-residuos/",
+        "https://reducereutilizarecicla.org/como-reciclar-pilas/",
+        "https://reducereutilizarecicla.org/como-reciclar-capsulas-de-cafe/",
+        "https://reducereutilizarecicla.org/reciclando-cd/",
+        "https://reducereutilizarecicla.org/donde-tirar-neumaticos/",
+        "https://reducereutilizarecicla.org/como-reciclar-una-canasta-de-mimbre/",
+        "https://reducereutilizarecicla.org/como-funciona-el-contenedor-amarillo/",
+        "https://reducereutilizarecicla.org/donde-se-tiran-los-vapers/",
+        "https://reducereutilizarecicla.org/cuanto-tarda-una-bolsa-biodegradable-en-degradarse/",
+        "https://reducereutilizarecicla.org/donde-se-reciclan-los-juguetes/",
+        "https://reducereutilizarecicla.org/objetos-que-se-pueden-reutilizar/",
+        "https://reducereutilizarecicla.org/la-parafina-se-puede-reutilizar/",
+        "https://reducereutilizarecicla.org/planta-de-reciclaje-de-papel/",
+        "https://reducereutilizarecicla.org/como-saber-si-un-envase-es-reciclable/",
+        "https://reducereutilizarecicla.org/reutilizar-vasos-de-vela/",
+        "https://reducereutilizarecicla.org/bolsas-frio-calor/",
+        "https://reducereutilizarecicla.org/reciclar-y-reutilizar-materiales-de-construccion/",
+        "https://reducereutilizarecicla.org/que-es-exactamente-el-pet/",
+        "https://reducereutilizarecicla.org/tipos-de-reciclaje/",
+        "https://reducereutilizarecicla.org/que-hacer-con-palets-reciclados/",
+        "https://reducereutilizarecicla.org/vertederos-controlados/",
+        "https://reducereutilizarecicla.org/donde-tirar-escombros/",
+        "https://reducereutilizarecicla.org/como-reciclar-los-residuos-de-ps-poliestireno/",
+        "https://reducereutilizarecicla.org/tirar-la-basura-sin-bolsas/",
+        "https://reducereutilizarecicla.org/tirar-el-palo-de-la-fregona/",
+        "https://reducereutilizarecicla.org/la-mejor-manera-de-reciclar-una-pala-de-padel/",
+        "https://reducereutilizarecicla.org/sabes-donde-tirar-las-llantas-viejas-de-un-coche/",
+        "https://reducereutilizarecicla.org/sabes-donde-tirar-el-arbol-de-navidad/",
+        "https://reducereutilizarecicla.org/clavos-tornillos-herramientas-donde-tirar-hierro/",
+        "https://reducereutilizarecicla.org/donde-tirar-un-secador-de-pelo-contenedor-o-punto-limpio/",
+        "https://reducereutilizarecicla.org/donde-tirar-electrodomesticos/",
+        "https://reducereutilizarecicla.org/donde-puedo-tirar-ramas-de-arboles/",
+        "https://reducereutilizarecicla.org/donde-tirar-escombros/",
+        "https://reducereutilizarecicla.org/donde-se-tira-el-muerdago-quemado/",
+        "https://reducereutilizarecicla.org/sandalias-caucho-reciclado-neumaticos/",
+        "https://reducereutilizarecicla.org/ideas-para-reciclar-aspas-de-ventilador-de-techo/",
+        "https://reducereutilizarecicla.org/reciclar-sacos-dormir/",
+        "https://reducereutilizarecicla.org/reciclar-sillas-playa/",
+        "https://reducereutilizarecicla.org/donde-tirar-antipolillas/",
+        "https://reducereutilizarecicla.org/que-hacer-con-los-juguetes-viejos/",
+        "https://reducereutilizarecicla.org/como-utilizar-las-mascarillas-y-el-gel-hidroalcoholico-en-la-playa/",
+        "https://reducereutilizarecicla.org/ideas-para-reciclar-un-ventilador-de-pie/",
+        "https://reducereutilizarecicla.org/donde-tirar-gasoil/",
+        "https://reducereutilizarecicla.org/donde-puedo-tirar-basura-electronica/",
+        "https://reducereutilizarecicla.org/donde-tirar-agujas/",
+        "https://reducereutilizarecicla.org/donde-tirar-residuos-peligrosos/",
+        "https://reducereutilizarecicla.org/donde-tirar-los-cables/",
+        "https://reducereutilizarecicla.org/donde-tirar-bicicletas/",
+        "https://reducereutilizarecicla.org/donde-tirar-maletas/",
+        "https://reducereutilizarecicla.org/como-reciclar-una-pantalla/",
+        "https://reducereutilizarecicla.org/donde-tirar-ropa-usada/"
+    ],
+    header_template=header_template
+)
+
 data=loader.load()
+
 # split documents
 text_splitter = RecursiveCharacterTextSplitter(
     chunk_size=1024,

requirements.txt

@@ -9,4 +9,5 @@ chromadb
 gradio
 beautifulsoup4
 tensorflow==2.13.0
-tenacity==8.3.0
+tenacity==8.3.0
+fake_useragent