Upload 3 files

collect_data.py

@@ -0,0 +1,312 @@
+import requests
+import pandas as pd
+from bs4 import BeautifulSoup
+import pysbd
+from datetime import datetime, timedelta
+
+def extract_div_contents_with_additional_columns(url, log_date):
+    response = requests.get(url)
+    if response.status_code != 200:
+        return pd.DataFrame(columns=['log_date', 'title', 'text_url', 'deletion_discussion', 'label', 'confirmation', 'verdict', 'discussion'])
+
+    soup = BeautifulSoup(response.content, 'html.parser')
+    div_classes = ['boilerplate afd vfd xfd-closed', 'boilerplate afd vfd xfd-closed archived mw-archivedtalk']
+    divs = []
+    for div_class in div_classes:
+        divs.extend(soup.find_all('div', class_=div_class))
+    url_fragment = url.split('#')[-1].replace('_', ' ')
+    data = []
+    for div in divs:
+        title_tag = div.find('a')
+        if title_tag:
+            title_span = div.find('span', {'data-mw-comment-start': True})
+            if title_span:
+                title_anchor = title_span.find_next_sibling('a')
+                if title_anchor:
+                    title = title_anchor.text
+                    text_url = 'https://en.wikipedia.org' + title_anchor['href']
+            else:
+                title = title_tag.text
+                text_url = 'https://en.wikipedia.org' + title_tag['href']
+                
+            deletion_discussion = div.prettify()
+
+            # Extract label
+            label = ''
+            verdict_tag = div.find('p')
+            if verdict_tag:
+                label_b_tag = verdict_tag.find('b')
+                if label_b_tag:
+                    label = verdict_tag.prettify()
+
+            # Extract confirmation
+            confirmation = ''
+            discussion_tag = div.find('dd').find('i')
+            if discussion_tag:
+                confirmation_b_tag = discussion_tag.find('b')
+                if confirmation_b_tag:
+                    confirmation = discussion_tag.prettify()
+
+        
+            parts = deletion_discussion.split('<div class="mw-heading mw-heading3">')
+            discussion = parts[0] if len(parts) > 0 else ''
+            verdict = '<div class="mw-heading mw-heading3">' + parts[1] if len(parts) > 1 else ''
+            data.append([log_date, title, text_url, deletion_discussion, label, confirmation, discussion, verdict])
+    df = pd.DataFrame(data, columns=['log_date', 'title', 'text_url', 'deletion_discussion', 'label', 'confirmation', 'verdict', 'discussion'])
+    return df
+
+
+def extract_div_contents_from_url(url,date):
+    response = requests.get(url)
+    if response.status_code != 200:
+        print(f"Error: Received status code {response.status_code} for URL: {url}")
+        return pd.DataFrame(columns=['date','title', 'text_url', 'deletion_discussion', 'label', 'confirmation', 'discussion', 'verdict'])
+
+    soup = BeautifulSoup(response.content, 'html.parser')
+    div_classes = ['boilerplate afd vfd xfd-closed', 'boilerplate afd vfd xfd-closed archived mw-archivedtalk']
+    divs = []
+    for div_class in div_classes:
+        divs.extend(soup.find_all('div', class_=div_class))
+    url_fragment = url.split('#')[-1].replace('_', ' ')
+    log_date = url.split('/')[-1]
+
+
+    data = []
+    for div in divs:
+        try:
+            title = None
+            text_url = None
+            title_tag = div.find('a')
+            if title_tag:
+                title_span = div.find('span', {'data-mw-comment-start': True})
+                if title_span:
+                    title_anchor = title_span.find_next_sibling('a')
+                    if title_anchor:
+                        title = title_anchor.text
+                        text_url = 'https://en.wikipedia.org' + title_anchor['href']
+                else:
+                    title = title_tag.text
+                    text_url = 'https://en.wikipedia.org' + title_tag['href']
+
+            if title == 'talk page' or title is None:
+                heading_tag = div.find('div', class_='mw-heading mw-heading3')
+                if heading_tag:
+                    title_tag = heading_tag.find('a')
+                    if title_tag:
+                        title = title_tag.text
+                        text_url = 'https://en.wikipedia.org' + title_tag['href']
+
+            if not title:
+                continue
+            if title.lower() != url_fragment.lower():
+                continue  
+            deletion_discussion = div.prettify()
+            label = ''
+            verdict_tag = div.find('p')
+            if verdict_tag:
+                label_b_tag = verdict_tag.find('b')
+                if label_b_tag:
+                    label = label_b_tag.text.strip()
+            confirmation = ''
+            discussion_tag = div.find('dd')
+            if discussion_tag:
+                discussion_tag_i = discussion_tag.find('i')
+                if discussion_tag_i:
+                    confirmation_b_tag = discussion_tag_i.find('b')
+                    if confirmation_b_tag:
+                        confirmation = confirmation_b_tag.text.strip()
+            parts = deletion_discussion.split('<div class="mw-heading mw-heading3">')
+            discussion = parts[0] if len(parts) > 0 else ''
+            verdict = '<div class="mw-heading mw-heading3">' + parts[1] if len(parts) > 1 else ''
+
+            data.append([date,title, text_url, deletion_discussion, label, confirmation, verdict, discussion])
+        except Exception as e:
+            print(f"Error processing div: {e}")
+            continue
+
+    df = pd.DataFrame(data, columns=['date', 'title', 'text_url', 'deletion_discussion', 'label', 'confirmation', 'discussion', 'verdict'])
+    return df
+
+
+
+
+def extract_div_contents_from_url_new(url,date):
+    response = requests.get(url)
+    if response.status_code != 200:
+        print(f"Error: Received status code {response.status_code} for URL: {url}")
+        return pd.DataFrame(columns=['date', 'title', 'text_url', 'deletion_discussion', 'label', 'confirmation', 'discussion', 'verdict'])
+
+    soup = BeautifulSoup(response.content, 'html.parser')
+    div_classes = ['boilerplate afd vfd xfd-closed', 'boilerplate afd vfd xfd-closed archived mw-archivedtalk',"mw-heading mw-heading3"]
+    divs = []
+
+    for div_class in div_classes:
+        divs.extend(soup.find_all('div', class_=div_class))
+
+    url_fragment = url.split('#')[-1].replace('_', ' ')
+    log_date = url.split('/')[-1]
+
+    data = []
+    for i, div in enumerate(divs):
+        try:
+            title = None
+            text_url = None
+            title_tag = div.find('a')
+            if title_tag:
+                title_span = div.find('span', {'data-mw-comment-start': True})
+                if title_span:
+                    title_anchor = title_span.find_next_sibling('a')
+                    if title_anchor:
+                        title = title_anchor.text
+                        text_url = 'https://en.wikipedia.org' + title_anchor['href']
+                else:
+                    title = title_tag.text
+                    text_url = 'https://en.wikipedia.org' + title_tag['href']
+
+            if title == 'talk page' or title is None:
+                heading_tag = div.find('div', class_='mw-heading mw-heading3')
+                if heading_tag:
+                    title_tag = heading_tag.find('a')
+                    if title_tag:
+                        title = title_tag.text
+                        text_url = 'https://en.wikipedia.org' + title_tag['href']
+
+            if not title:
+                continue
+            if title.lower() != url_fragment.lower():
+                continue
+
+            next_div = div.find_next('div', class_='mw-heading mw-heading3')
+            deletion_discussion = ''
+            sibling = div.find_next_sibling()
+            while sibling and sibling != next_div:
+                deletion_discussion += str(sibling)
+                sibling = sibling.find_next_sibling()
+
+            label = ''
+            verdict_tag = div.find('p')
+            if verdict_tag:
+                label_b_tag = verdict_tag.find('b')
+                if label_b_tag:
+                    label = label_b_tag.text.strip()
+            confirmation = ''
+            discussion_tag = div.find('dd')
+            if discussion_tag:
+                discussion_tag_i = discussion_tag.find('i')
+                if discussion_tag_i:
+                    confirmation_b_tag = discussion_tag_i.find('b')
+                    if confirmation_b_tag:
+                        confirmation = confirmation_b_tag.text.strip()
+            parts = deletion_discussion.split('<div class="mw-heading mw-heading3">')
+            discussion = parts[0] if len(parts) > 0 else ''
+            verdict = '<div class="mw-heading mw-heading3">' + parts[1] if len(parts) > 1 else ''
+
+            data.append([date, title, text_url, deletion_discussion, label, confirmation, verdict, discussion])
+        except Exception as e:
+            print(f"Error processing div: {e}")
+            continue
+
+    df = pd.DataFrame(data, columns=['date', 'title', 'text_url', 'deletion_discussion', 'label', 'confirmation', 'discussion', 'verdict'])
+    return df
+
+def extract_label(label_html):
+    soup = BeautifulSoup(label_html, 'html.parser')
+    b_tag = soup.find('b')
+    return b_tag.text.strip() if b_tag else ''
+
+def process_labels(df):
+    df['proper_label'] = df['label'].apply(extract_label)
+    return df
+
+def extract_confirmation(confirmation_html):
+    soup = BeautifulSoup(confirmation_html, 'html.parser')
+    b_tag = soup.find('span', {'style': 'color:red'}).find('b')
+    return b_tag.text.strip() if b_tag else ''
+
+def process_confirmations(df):
+    df['confirmation'] = df['confirmation'].apply(extract_confirmation)
+    return df
+
+def extract_post_links_text(discussion_html):
+    split_point = '<span class="plainlinks">'
+    if split_point in discussion_html:
+        parts = discussion_html.split(split_point)
+        if len(parts) > 1:
+            return parts[1]
+    return discussion_html
+
+def process_discussion(df):
+
+    df['discussion_cleaned'] = df['discussion'].apply(extract_post_links_text)
+    return df
+
+def html_to_plaintext(html_content):
+    soup = BeautifulSoup(html_content, 'html.parser')
+    for tag in soup.find_all(['p', 'li', 'dd', 'dl']):
+        tag.insert_before('\n')
+        tag.insert_after('\n')
+    for br in soup.find_all('br'):
+        br.replace_with('\n')
+
+    text = soup.get_text(separator=' ', strip=True)
+    text = '\n'.join([line.strip() for line in text.splitlines() if line.strip() != ''])
+
+    return text
+
+def process_html_to_plaintext(df):
+    df['discussion_cleaned'] = df['discussion_cleaned'].apply(html_to_plaintext)
+    return df
+
+def split_text_into_sentences(text):
+    seg = pysbd.Segmenter(language="en", clean=False)
+    sentences = seg.segment(text)
+    return ' '.join(sentences[1:])
+
+def process_split_text_into_sentences(df):
+    df['discussion_cleaned'] = df['discussion_cleaned'].apply(split_text_into_sentences)
+    return df
+
+def process_data(url,date):
+    df = extract_div_contents_from_url(url,date)
+    #print('Discussion: ',df.discussion.tolist())
+    if df.discussion.tolist() == []:
+      #print('Empty Discussion')
+      df = extract_div_contents_from_url_new(url,date)
+    #print(df.head())
+    df = process_discussion(df)
+    #print(df.at[0,'discussion'])
+    df = process_html_to_plaintext(df)
+    df = process_split_text_into_sentences(df)
+    if not df.empty:
+        return df
+    else:
+        return 'Empty DataFrame'
+
+def collect_deletion_discussions(start_date, end_date):
+    base_url = 'https://en.wikipedia.org/wiki/Wikipedia:Articles_for_deletion/Log/'
+    all_data = pd.DataFrame()
+
+    current_date = start_date
+    while current_date <= end_date:
+        try:
+            print(f"Processing {current_date.strftime('%Y-%B-%d')}")
+            date_str = current_date.strftime('%Y_%B_%d')
+            url = base_url + date_str
+            log_date = current_date.strftime('%Y-%m-%d')
+
+            df = extract_div_contents_with_additional_columns(url, log_date)
+            if not df.empty:
+                df = process_labels(df)
+                df = process_confirmations(df)
+                df = process_discussion(df)
+                df = process_html_to_plaintext(df)
+                df = process_split_text_into_sentences(df)
+                all_data = pd.concat([all_data, df], ignore_index=True)
+
+            current_date += timedelta(days=1)
+        except Exception as e:
+            print(f"Error processing {current_date.strftime('%Y-%B-%d')}: {e}")
+            current_date += timedelta(days=1)
+            continue
+
+    return all_data

process_data.py

@@ -1,6 +1,6 @@
 import pandas as pd
 from datetime import datetime
-from wide_analysis.data.collect_data import collect_deletion_discussions, process_data
+from collect_data import collect_deletion_discussions, process_data
 
 
 label_mapping = {

requirements.txt

@@ -1,9 +1,9 @@
-beautifulsoup4==4.12.3
-datasets==2.21.0
-gradio==5.8.0
-pandas==2.0.1
-pysbd==0.3.4
-Requests==2.32.3
-torch==2.2.1
-transformers==4.44.1
-wide_analysis==0.3.3
+beautifulsoup4
+datasets
+gradio
+pandas
+pysbd
+Requests
+torch
+transformers
+wide_analysis