Upload data_prep.py

data_prep.py

@@ -82,11 +82,7 @@ def extract_div_contents_from_url(url):
     return df
 
 
-import requests
-import pandas as pd
-from bs4 import BeautifulSoup
-
-def extract_div_contents_from_url_new(url, date):
+def extract_div_contents_from_url_new(url):
     response = requests.get(url)
     if response.status_code != 200:
         print(f"Error: Received status code {response.status_code} for URL: {url}")
@@ -154,15 +150,15 @@ def extract_div_contents_from_url_new(url, date):
                     if confirmation_b_tag:
                         confirmation = confirmation_b_tag.text.strip()
             parts = deletion_discussion.split('<div class="mw-heading mw-heading3">')
-            discussion = parts[-1] if len(parts) > 0 else ''
+            discussion = parts[0] if len(parts) > 0 else ''
             verdict = '<div class="mw-heading mw-heading3">' + parts[1] if len(parts) > 1 else ''
 
-            data.append([date, title, text_url, deletion_discussion, label, confirmation, verdict, discussion])
+            data.append([ title, text_url, deletion_discussion, label, confirmation, verdict, discussion])
         except Exception as e:
             print(f"Error processing div: {e}")
             continue
 
-    df = pd.DataFrame(data, columns=['date', 'title', 'text_url', 'deletion_discussion', 'label', 'confirmation', 'discussion', 'verdict'])
+    df = pd.DataFrame(data, columns=[ 'title', 'text_url', 'deletion_discussion', 'label', 'confirmation', 'discussion', 'verdict'])
     return df
 
 def extract_post_links_text(discussion_html):
@@ -206,13 +202,13 @@ def process_split_text_into_sentences(df):
 
 def process_data(url):
     df = extract_div_contents_from_url(url)
-    #check if df is empty
-    if df.empty:
+    if df.at[0,'discussion'] == '':
        df = extract_div_contents_from_url_new(url)
+    #print(df.head())
     df = process_discussion(df)
+    print(df.at[0,'discussion'])
     df = process_html_to_plaintext(df)
     df = process_split_text_into_sentences(df)
-    #if not empty
     if not df.empty:
         return df.at[0,'title']+ ' : '+df.at[0, 'discussion_cleaned']
     else: