initial commit

app.py

@@ -0,0 +1,41 @@
+import data_prep
+import model_predict
+import gradio as gr
+
+# Dictionary of model names and corresponding display names
+model_dict = {
+    "bert-base": "research-dump/bert-base-uncased_deletion_multiclass_complete_Final",
+    "bert-large": "research-dump/bert-large-uncased_deletion_multiclass_complete_final",
+    "roberta-base": "research-dump/roberta-base_deletion_multiclass_complete_final"
+}
+
+def process_url(url, model_key):
+    # Get the actual model path from the model_dict
+    model_name = model_dict[model_key]
+
+    # Process the text from the URL
+    processed_text = data_prep.process_data(url)
+    
+    # Predict the labels and their probabilities
+    final_scores = model_predict.predict_text(processed_text, model_name)
+    
+    # Find the label with the highest probability
+    highest_prob_label = max(final_scores, key=final_scores.get)
+    highest_prob = final_scores[highest_prob_label]
+    
+    # Create progress bar style output for all labels
+    progress_bars = {label: score for label, score in final_scores.items()}
+    
+    return highest_prob_label, highest_prob, progress_bars
+
+# Define the interface for the Gradio app
+url_input = gr.Textbox(label="URL")
+model_name_input = gr.Dropdown(label="Model Name", choices=list(model_dict.keys()), value=list(model_dict.keys())[0])
+outputs = [
+    gr.Textbox(label="Label with Highest Probability"),
+    gr.Textbox(label="Probability"),
+    gr.JSON(label="All Labels and Probabilities")
+]
+
+demo = gr.Interface(fn=process_url, inputs=[url_input, model_name_input], outputs=outputs)
+demo.launch()

data_prep.py

@@ -0,0 +1,131 @@
+import requests
+import pandas as pd
+from bs4 import BeautifulSoup
+import pandas as pd
+from datetime import datetime
+
+
+def extract_div_contents_from_url(url):
+    response = requests.get(url)
+    if response.status_code != 200:
+        return pd.DataFrame(columns=['title', 'text_url', 'deletion_discussion', 'label', 'confirmation', 'discussion', 'verdict'])
+
+    soup = BeautifulSoup(response.content, 'html.parser')
+    divs = soup.find_all('div', class_='boilerplate afd vfd xfd-closed archived mw-archivedtalk')
+
+    # Extract the title fragment from the URL
+    url_fragment = url.split('#')[-1].replace('_', ' ')
+
+    data = []
+    for div in divs:
+        try:
+            title = None
+            text_url = None
+
+            # Extract title and text_url
+            title_tag = div.find('a')
+            if title_tag:
+                title_span = div.find('span', {'data-mw-comment-start': True})
+                if title_span:
+                    title_anchor = title_span.find_next_sibling('a')
+                    if title_anchor:
+                        title = title_anchor.text
+                        text_url = 'https://en.wikipedia.org' + title_anchor['href']
+                else:
+                    title = title_tag.text
+                    text_url = 'https://en.wikipedia.org' + title_tag['href']
+
+            if title == 'talk page' or title is None:
+                heading_tag = div.find('div', class_='mw-heading mw-heading3')
+                if heading_tag:
+                    title_tag = heading_tag.find('a')
+                    if title_tag:
+                        title = title_tag.text
+                        text_url = 'https://en.wikipedia.org' + title_tag['href']
+
+            if title != url_fragment:
+                continue  # Skip if the title does not match the URL fragment
+
+            deletion_discussion = div.prettify()
+
+            # Extract label
+            label = ''
+            verdict_tag = div.find('p')
+            if verdict_tag:
+                label_b_tag = verdict_tag.find('b')
+                if label_b_tag:
+                    label = label_b_tag.text.strip()
+
+            # Extract confirmation
+            confirmation = ''
+            discussion_tag = div.find('dd').find('i')
+            if discussion_tag:
+                confirmation_b_tag = discussion_tag.find('b')
+                if confirmation_b_tag:
+                    confirmation = confirmation_b_tag.text.strip()
+
+            # Split deletion_discussion into discussion and verdict
+            parts = deletion_discussion.split('<div class="mw-heading mw-heading3">')
+            discussion = parts[0] if len(parts) > 0 else ''
+            verdict = '<div class="mw-heading mw-heading3">' + parts[1] if len(parts) > 1 else ''
+
+            data.append([title, text_url, deletion_discussion, label, confirmation, verdict, discussion])
+        except Exception as e:
+            print(f"Error processing div: {e}")
+            continue
+
+    df = pd.DataFrame(data, columns=['title', 'text_url', 'deletion_discussion', 'label', 'confirmation', 'verdict', 'discussion'])
+    df = df[['title','discussion','verdict','label']]
+    return df
+
+
+def extract_post_links_text(discussion_html):
+    split_point = '<span class="plainlinks">'
+    if split_point in discussion_html:
+        parts = discussion_html.split(split_point)
+        if len(parts) > 1:
+            return parts[1]
+    return discussion_html
+
+def process_discussion(df):
+    df['discussion_cleaned'] = df['verdict'].apply(extract_post_links_text)
+    return df
+
+def html_to_plaintext(html_content):
+    soup = BeautifulSoup(html_content, 'html.parser')
+    for tag in soup.find_all(['p', 'li', 'dd', 'dl']):
+        tag.insert_before('\n')
+        tag.insert_after('\n')
+    for br in soup.find_all('br'):
+        br.replace_with('\n')
+
+    text = soup.get_text(separator=' ', strip=True)
+    text = '\n'.join([line.strip() for line in text.splitlines() if line.strip() != ''])
+
+    return text
+def process_html_to_plaintext(df):
+    df['discussion_cleaned'] = df['discussion_cleaned'].apply(html_to_plaintext)
+    df = df[['title', 'discussion_cleaned', 'label']]
+    return df
+
+import pysbd
+def split_text_into_sentences(text):
+    seg = pysbd.Segmenter(language="en", clean=False)
+    sentences = seg.segment(text)
+    return ' '.join(sentences[1:])
+def process_split_text_into_sentences(df):
+    df['discussion_cleaned'] = df['discussion_cleaned'].apply(split_text_into_sentences)
+    return df
+
+
+
+def process_data(url):
+    df = extract_div_contents_from_url(url)
+    df = process_discussion(df)
+    df = process_html_to_plaintext(df)
+    df = process_split_text_into_sentences(df)
+    #if not empty
+    if not df.empty:
+        return df.at[0,'title']+ ' : '+df.at[0, 'discussion_cleaned']
+    else:
+        return 'Empty DataFrame'

model_predict.py

@@ -0,0 +1,28 @@
+#using pipeline to predict the input text
+from transformers import pipeline
+import torch
+
+label_mapping = {
+    'delete': [0, 'LABEL_0'],
+    'keep': [1, 'LABEL_1'],
+    'merge': [2, 'LABEL_2'],
+    'no consensus': [3, 'LABEL_3'],
+    'speedy keep': [4, 'LABEL_4'],
+    'speedy delete': [5, 'LABEL_5'],
+    'redirect': [6, 'LABEL_6'],
+    'withdrawn': [7, 'LABEL_7']
+}
+
+def predict_text(text, model_name):
+    model = pipeline("text-classification", model=model_name, return_all_scores=True)
+    results = model(text)
+    final_scores = {key: 0.0 for key in label_mapping}
+    
+    for result in results[0]:
+        for key, value in label_mapping.items():
+            if result['label'] == value[1]:
+                final_scores[key] = result['score']
+                break
+    
+    return final_scores
+

requirements.txt

@@ -0,0 +1,7 @@
+requests
+beautifulsoup4
+pandas
+numpy
+pysbd
+transformers
+torch