Update app.py

app.py

@@ -4,7 +4,6 @@ from llama_index.llms.huggingface import HuggingFaceInferenceAPI
 from llama_index.embeddings.huggingface import HuggingFaceEmbedding
 from llama_index.core import Settings
 from youtube_transcript_api import YouTubeTranscriptApi
-from langchain_community.document_loaders import WebBaseLoader
 import shutil
 import os
 import time
@@ -40,13 +39,8 @@ def data_ingestion():
     index.storage_context.persist(persist_dir=PERSIST_DIR)
 
 def remove_old_files():
-    # Specify the directory path you want to clear
     directory_path = "data"
-
-    # Remove all files and subdirectories in the specified directory
     shutil.rmtree(directory_path)
-
-    # Recreate an empty directory if needed
     os.makedirs(directory_path)
 
 def extract_transcript_details(youtube_video_url):
@@ -64,18 +58,6 @@ def extract_transcript_details(youtube_video_url):
     except Exception as e:
         st.error(e)
 
-def get_url_text(url_link):
-    try:
-        loader = WebBaseLoader(url_link)
-        loader.requests_per_second = 1
-        docs = loader.aload()
-        extracted_text = ""
-        for page in docs:
-            extracted_text += page.page_content
-        return extracted_text
-    except Exception as e:
-        st.error(e)
-
 def handle_query(query):
     storage_context = StorageContext.from_defaults(persist_dir=PERSIST_DIR)
     index = load_index_from_storage(storage_context)
@@ -134,13 +116,6 @@ with st.sidebar:
                 print(filepath)
                 with open(filepath, "wb") as f:
                     f.write(uploaded_file.getbuffer())
-
-            if uploaded_url:
-                url_text = get_url_text(uploaded_url)
-                print(url_text)
-                with open("data/url_text.txt", "w") as fil:
-                    fil.write(url_text)
-                    print(os.listdir("data"))
         
             if video_url:
                 extracted_text = extract_transcript_details(video_url)