collect_data_wikinews.py

import requests
from bs4 import BeautifulSoup
import pandas as pd
import pysbd
import re


################################
# Year based data collection ###
################################

def get_soup(url):
    response = requests.get(url)
    response.raise_for_status()  
    return BeautifulSoup(response.text, 'html.parser')

def html_to_plaintext(html_content):
    soup = BeautifulSoup(html_content, 'html.parser')
    for tag in soup.find_all(['p', 'li', 'dd', 'dl']):
        tag.insert_before('\n')
        tag.insert_after('\n')
    for br in soup.find_all('br'):
        br.replace_with('\n')

    text = soup.get_text(separator=' ', strip=True)
    text = '\n'.join([line.strip() for line in text.splitlines() if line.strip() != ''])
    return text

def extract_fallback_discussion(html_content):
    soup = BeautifulSoup(html_content, 'html.parser')
    discussion_parts = []
    for element in soup.find_all(['p', 'li', 'dd', 'ol'], recursive=False):
        discussion_parts.append(element.get_text(separator=' ', strip=True))
    return ' '.join(discussion_parts).strip()

def process_html_to_plaintext(df):
    if df.empty:
        return df
    df['discussion_cleaned'] = df['discussion'].apply(html_to_plaintext)
    for index, row in df.iterrows():
        if not row['discussion_cleaned'].strip():
            df.at[index, 'discussion_cleaned'] = extract_fallback_discussion(row['discussion_uncleaned']) 
    return df

def extract_outcome_from_div(div):
    try:
        result_phrase = div.find(text=re.compile(r'The result was to'))
        if result_phrase:
            result = result_phrase.find_next('b')
            if result:
                outcome_text = result.text.strip()
                if outcome_text.lower() == "please do not modify it":
                    return extract_following_sentence(div) or 'unknown'
                elif validate_outcome(outcome_text):
                    return outcome_text  
        li_outcome = div.find('li')
        if li_outcome and li_outcome.find('b'):
            outcome_text = li_outcome.find('b').text.strip()
            if outcome_text.lower() == "please do not modify it":
                return extract_following_sentence(div) or 'unknown'
            elif validate_outcome(outcome_text):
                return outcome_text

        dl_outcome = div.find('dl')
        if dl_outcome and dl_outcome.find('b'):
            outcome_text = dl_outcome.find('b').text.strip()
            if outcome_text.lower() == "please do not modify it":
                return extract_following_sentence(div) or 'unknown'
            elif validate_outcome(outcome_text):
                return outcome_text

        outcome_italic = div.find('dd')
        if outcome_italic and outcome_italic.find('i'):
            outcome_text = outcome_italic.find('i').get_text(strip=True)
            if outcome_text.lower() == "please do not modify it":
                return extract_following_sentence(div) or 'unknown'
            elif validate_outcome(outcome_text):
                return outcome_text
        return extract_following_sentence(div) or 'unknown'

    except Exception as e:
        print(f"Error extracting outcome: {e}")
        return 'unknown'


def extract_following_sentence(div):
    try:
        phrases = [
            "No further edits should be made to this discussion",
            "Please do not add any more comments and votes to this request",
            "No further edits should be made to this discussion."
        ]
        
        for phrase in phrases:
            phrase_location = div.find(text=re.compile(phrase))
            if phrase_location:
                following_text = ""
                for sibling in phrase_location.find_all_next(string=True):
                    if "Please do not modify it" in sibling:
                        continue
                    following_text += sibling.strip() + " "
                    if "." in sibling:
                        break
                sentence = following_text.split('.')[0].strip()
                if validate_outcome(sentence):
                    return sentence
        
        return None 

    except Exception as e:
        print(f"Error extracting following sentence: {e}")
        return None

def validate_outcome(outcome_text):
    label_mapping = {
        'delete': [
            'delete', 'delete ... unanimous', 'deleted', 'deleted as abandoned',
            'speedy delete', 'Delete', 'delete as redundant to existing template',
            'delete as unlikely to be used', 'delete but no prejudice against recreation when needed',
            'delete after Ottawahitech chose not to provide a rationale',
            'Delete, with no objection to recreation when needed.', 'Deleted', 
            'delete the Cigarette redirect and keep the NHS redirect.', 'Delete all articles', 'Tentatively sending through the usual abandonment process',
            'Delete all articles','This was completed already.'
        ],
'speedy delete': [ 

            'speedy delete', 'speedy deleted', 'speedy deleted test page', 'Speedy-deleted', 'Speedy deleted', 'Speedy-deleted, no meaningful content',
            'Speeded as "old prep"', 'Speedied as "old prep"  -- Pi zero ( talk ) 23:42, 10 February 2020 (UTC)   [  reply  ]  __DTELLIPSISBUTTON__{"threadItem":{"timestamp":"2020-02-10T23:42:00'
],

        'keep': [
             'keep',
            'Do not undelete. The content should be kept by the author off-wiki, and can be included as a part of another story that is current',
            'Personal details have been redacted and hidden from public view together with a NOINDEX flag',

            ],
        'redirect': [
            'soft redirect'
        ],
        'merge': [
            'convert near-clone of mainspace article to use {{topic cat}}; apply {{correction}} to mainspace article'
        ],
        'no_consensus': [
            'No consensus to delete. However, there clearly is a consensus that if we are to have this template, we aren\'t to use it in its present form.',
            'no consensus', 'No consensus',
            "At this time, it's unclear if there's a consensus to keep but abundantly clear there isn't one to delete."
        ],
        'comment': [
            'Remove', 'SVT', 'withdraw the deletion request', 'On consideration, speedied as unused and lacking fair-use rationale',
            'Moved to userspace', 'Withdrawn to allow interview re-focus','More userspace drafts       This is the second batch of a large number of draft articles in userspace',
            'This was completed already ', 'Do not undelete. The content should be kept by the author off-wiki, and can be included as a part of another story that is current',

        ],
        'withdrawn': ['Withdrawn to allow interview re-focus', 
        ]
    }

    

    outcome_to_label = {outcome.lower(): label for label, outcomes in label_mapping.items() for outcome in outcomes}
    return outcome_to_label.get(outcome_text.lower(), 'unknown')


def update_unknown_outcomes(df):
    base_url = "https://en.wikinews.org/w/index.php?title="

    for i in df.index:
        if df.at[i, 'outcome'] == 'unknown':
            title = df.at[i, 'title'].replace(" ", "_") 
            url = f"{base_url}{title}&action=edit&redlink=1"
            print(f"Checking page: {url}")

            try:
                response = requests.get(url)
                if response.status_code == 200:
                    page_soup = BeautifulSoup(response.content, 'html.parser')
                    
                    # Look for the specific warning div
                    warning_div = page_soup.find('div', class_='cdx-message cdx-message--block cdx-message--warning mw-warning-with-logexcerpt')
                    if warning_div:
                        df.at[i, 'outcome'] = 'delete'
                    else:
                        df.at[i, 'outcome'] = 'keep'
                else:
                    print(f"Failed to retrieve page: {url}")
            
            except Exception as e:
                print(f"Error accessing {url}: {e}")

    return df

    
def collect_wikinews_deletions(years=None):
    base_url = 'https://en.wikinews.org/wiki/Wikinews:Deletion_requests/Archives'
    response = requests.get(base_url)
    if response.status_code != 200:
        print("Failed to retrieve the archive page.")
        return None
    
    soup = get_soup(base_url)
    titles = []
    text_urls = []
    outcomes = []
    deletion_discussions = []  
    discussion_uncleaned = [] 
    year_links = []
    for a in soup.select('a[href^="/wiki/Wikinews:Deletion_requests/Archives/"]'):
        year_text = re.findall(r'\d{4}', a.get_text()) 
        if year_text:
            year_links.append((year_text[0], a['href']))
    if years:
        if len(years) == 1:
            start_year = end_year = years[0]
        elif len(years) == 2:
            start_year, end_year = min(years), max(years)
        else:
            print("Invalid years input. Provide one or two years.")
            return None
        year_links = [(year, link) for year, link in year_links if start_year <= int(year) <= end_year]
    for year, year_link in year_links:
        year_url = 'https://en.wikinews.org' + year_link
        print(f"Processing year: {year_url}")
        year_soup = get_soup(year_url)
        discussion_divs = year_soup.find_all('div', class_=lambda x: x and 'boilerplate metadata' in x)
        
        for div in discussion_divs:
            title_tag = div.find(['h2', 'h3'])
            if title_tag:
                link_tag = title_tag.find('a', title=True)
                if link_tag:
                    title = link_tag.get_text(strip=True)
                    titles.append(title)
                    text_url = year_url + '#' + link_tag['title'].replace(' ', '_')
                    text_urls.append(text_url)
                else:
                    titles.append(title_tag.get_text(strip=True))
                    text_urls.append(year_url)
            else:
                dl_tag = div.find('dl')
                if dl_tag and dl_tag.find('b'):
                    titles.append(dl_tag.find('b').get_text(strip=True))
                else:
                    titles.append('No title found')
                text_urls.append(year_url)
            deletion_discussions.append(div.prettify())
            discussion_uncleaned.append(div.prettify())
            outcome = extract_outcome_from_div(div)
            outcomes.append(outcome)

    df = pd.DataFrame({
        'title': titles,
        'url': text_urls,
        'outcome': outcomes,
        'discussion': deletion_discussions,
        'discussion_uncleaned': discussion_uncleaned  
    })


    df = process_html_to_plaintext(df)
    for i in df.index:
        if df.at[i,'outcome'] == 'Please do not modify it' or df.at[i,'outcome'] == 'Please do not modify it.':
            df.at[i,'outcome'] = extract_following_sentence(BeautifulSoup(df.at[i,'discussion_uncleaned'], 'html.parser')) or 'unknown'
    df['outcome'] = df['outcome'].apply(lambda x: validate_outcome(x) if x else 'unknown')
    df = update_unknown_outcomes(df)
    return df

def collect_wikinews(years=None):
    df = collect_wikinews_deletions(years=years)
    if df is None:
        print('Error collecting Wikinews deletions.')
        return None
    return df


##################################
## Ttitle based data collection ##
##################################

def html_to_plaintext(html_content):
    soup = BeautifulSoup(html_content, 'html.parser')
    for tag in soup.find_all(['p', 'li', 'dd', 'dl', 'ul']):
        tag.insert_before('\n')
        tag.insert_after('\n')
    for br in soup.find_all('br'):
        br.replace_with('\n')
    text = soup.get_text(separator=' ', strip=True)
    text = '\n'.join([line.strip() for line in text.splitlines() if line.strip() != ''])
    return text

def split_text_into_sentences(text):
    seg = pysbd.Segmenter(language="en", clean=False)
    sentences = seg.segment(text)
    return ' '.join(sentences)

def clean_discussion_tag(tag):
    for unwanted in tag.find_all(['span', 'img', 'a', 'div'], recursive=True):
        unwanted.decompose()
    return tag.get_text(separator=' ', strip=True)

def extract_outcome_from_text_elements(elements):
    consensus_keywords = [
        'Deleted', 'Delete', 'delete', 'deleted', 
        'kept', 'keep', 'Keep', 'Kept', 
        'merge', 'Merge', 'Not done', 'No consensus', 'no consensus', 'Done'
    ]
    for el in elements:
        b_tags = el.find_all('b')
        for b in b_tags:
            if b.text.strip() in consensus_keywords:
                return b.text.strip()
    return ''

def extract_discussion_section(soup, title):
    """Extracts discussion section, label, and cleaned text."""
    try:
        h3_id = title.replace(" ", "_") 
        h3_tag = soup.find('h3', {'id': h3_id})

        if not h3_tag:
            print(f"h3 tag with id '{h3_id}' not found.")
            return '', '', ''

        heading_div = h3_tag.parent

        if not heading_div:
            print("Parent div not found.")
            return '', '', ''

        next_heading_div = heading_div.find_next_sibling('div', class_='mw-heading mw-heading3')
        discussion_nodes = []
        for sibling in heading_div.next_siblings:
            if sibling == next_heading_div:
                break
            discussion_nodes.append(sibling)

        discussion_tags = []
        for node in discussion_nodes:
            if getattr(node, 'name', None) in ['p', 'ul', 'dl']:
                discussion_tags.append(node)

        if not discussion_tags:
            return '', '', ''

        label = extract_outcome_from_text_elements(discussion_tags)

        discussion_html_parts = [str(tag) for tag in discussion_tags]
        cleaned_parts = []
        for tag in discussion_tags:
            text = clean_discussion_tag(tag)
            if text:
                cleaned_parts.append(text)

        cleaned_discussion = ' '.join(cleaned_parts)
        discussion_html = '\n'.join(discussion_html_parts)
        return discussion_html, label, cleaned_discussion

    except Exception as e:
        print(f"Error processing title '{title}': {e}")
        import traceback
        traceback.print_exc()  
        return '', '', ''

def extract_div_from_title(title):
    base_url = 'https://en.wikinews.org/wiki/Wikinews:Deletion_requests'
    t = title.replace(' ', '_')
    url = base_url + '#' + t

    response = requests.get(url)
    if response.status_code != 200:
        return pd.DataFrame(columns=['title', 'text_url', 'discussion_url', 'discussion_cleaned', 'label'])

    soup = BeautifulSoup(response.content, 'html.parser')
    discussion_html, label, cleaned_discussion = extract_discussion_section(soup, title)
 
    text_url = base_url
    discussion_url = text_url + '#' + title.replace(' ', '_')

    df = pd.DataFrame([[title, text_url, discussion_url, cleaned_discussion, label]],
                      columns=['title', 'text_url', 'discussion_url', 'discussion_cleaned', 'label'])

    if label:
        df['label'] = df['label'].replace({
            'Deleted':'delete', 'Delete':'delete', 'delete':'delete', 'deleted':'delete', 
            'kept':'keep', 'keep':'keep', 'Keep':'keep', 'Kept':'keep', 
            'merge':'merge', 'Merge':'merge', 'Not done':'no_consensus', 
            'No consensus':'no_consensus', 'no consensus':'no_consensus', 'Done':'delete'
        })

    df['discussion_cleaned'] = df['discussion_cleaned'].apply(split_text_into_sentences)
    df = df.rename(columns={'discussion_cleaned':'discussion'})
    return df

########################
## Umbrella function  ##
########################

def collect_wikinews(mode, title=None, url ='', year=None):

    if mode == 'title':
        if not title:
            raise ValueError("Title is required for 'title' mode.")
        return extract_div_from_title(title)
    
    elif mode == 'url':
        if 'Archives' in url.split('/')[-2]:
            year = int(url.split('/')[-1].split('#')[0])
            print(f"Year extracted from URL: {year}")
            df = collect_wikinews_deletions(years=[year])
            #keep the row with the title only
            df = df[df['title'] == url.split('#')[-1].replace('_', ' ')]
            if df.empty:
                return pd.DataFrame(columns=['title', 'text_url', 'discussion_url', 'discussion_cleaned', 'label'])
            df = df[['title','url','discussion_cleaned','outcome']]
            df = df.rename(columns={'discussion_cleaned':'discussion'})   
            return df

        if not url:
            raise ValueError("URL is required for 'url' mode.")
        title = url.split('#')[-1].replace('_', ' ')
        print(f"Title extracted from URL: {title}")
        return extract_div_from_title(title)

    elif mode == 'year':
        if not year:
            raise ValueError("Year or year range is required for 'year' mode.")
        return collect_wikinews_deletions(years=year)

    else:
        raise ValueError("Invalid mode. Please specify 'title' or 'year' or 'url'.")

# year_df = collect_wikinews(mode='year', year=[2023]) 
# title_df = collect_wikinews(mode='title', title="NurMi spam") 

# print(year_df)
# print(title_df)