Upload app.py

app.py

@@ -0,0 +1,381 @@
+# app.py
+import streamlit as st
+from serpapi import GoogleSearch
+from sentence_transformers import SentenceTransformer, util
+from transformers import AutoTokenizer, AutoModelForSequenceClassification
+import torch
+import torch.nn.functional as F
+from urllib.parse import urlparse
+from bs4 import BeautifulSoup
+import requests
+import re
+import numpy as np
+import textwrap
+
+# ---------------- Settings ----------------
+st.set_page_config(page_title="Super Smart Fake News Detector — Advanced", page_icon="🧠", layout="centered")
+st.title("🧠 Super Smart Fake News Detector — Advanced")
+st.write("Dynamic verdict (TRUE / FAKE / INSUFFICIENT DATA) using Semantic Similarity, NLI (Entailment/Contradiction), and Credibility weighting.")
+
+# --- FULL POWER CONSTANTS ---
+NUM_RESULTS = 30         # Maximum number of search results to fetch
+TOP_K_FOR_VERDICT = 6    # Maximum number of top results to analyze
+# ----------------------------
+
+# ---------------- Caches / Model loaders ----------------
+@st.cache_resource
+def load_embedder():
+    # Force CPU to avoid device issues on Cloud
+    return SentenceTransformer('all-MiniLM-L6-v2', device='cpu')
+
+@st.cache_resource
+def load_nli_model():
+    # NLI model (roberta-large-mnli) - CPU mode
+    tok = AutoTokenizer.from_pretrained("roberta-large-mnli")
+    mdl = AutoModelForSequenceClassification.from_pretrained("roberta-large-mnli")
+    mdl.to("cpu")
+    return tok, mdl
+
+embedder = load_embedder()
+nli_tok, nli_model = load_nli_model()
+
+# ---------------- Utilities ----------------
+def domain_from_url(url):
+    try:
+        return urlparse(url).netloc.replace("www.", "")
+    except:
+        return url
+
+def pretty_pct(x):
+    return f"{int(x*100)}%"
+
+# ---------------- Rank-claim helpers (Wikipedia list check) ----------------
+ORDINAL_WORDS = {
+    "first":1, "second":2, "third":3, "fourth":4, "fifth":5, "sixth":6, "seventh":7, "eighth":8, "ninth":9, "tenth":10,
+    "eleventh":11, "twelfth":12, "thirteenth":13, "fourteenth":14, "fifteenth":15, "sixteenth":16, "seventeenth":17,
+    "eighteenth":18, "nineteenth":19, "twentieth":20
+}
+ROLE_KEYWORDS = ["prime minister", "prime-minister", "pm", "president", "chief minister", "cm", "governor", "chief justice"]
+
+def find_ordinal_and_role(text):
+    t = text.lower()
+    num = None
+    m = re.search(r'\b(\d{1,2})(?:st|nd|rd|th)?\b', t)
+    if m:
+        num = int(m.group(1))
+    else:
+        for w, n in ORDINAL_WORDS.items():
+            if re.search(r'\b' + re.escape(w) + r'\b', t):
+                num = n
+                break
+    role = None
+    for rk in ROLE_KEYWORDS:
+        if rk in t:
+            role = rk.replace('-', ' ')
+            break
+    return num, role
+
+def extract_person_candidate(text):
+    patterns = [
+        r"^([\w\s\.\-]{2,80}?)\s+is\s+the\b",
+        r"^([\w\s\.\-]{2,80}?)\s+is\s+(\d{1,2})",
+        r"is\s+([\w\s\.\-]{2,80}?)\s+the\s+\d{1,2}",
+        r"^([\w\s\.\-]{2,80}?)\s+was\s+the\b",
+    ]
+    for p in patterns:
+        mm = re.search(p, text, flags=re.IGNORECASE)
+        if mm:
+            name = mm.group(1).strip()
+            if len(name) > 1 and not re.match(r'^(it|he|she|they|this|that)$', name.lower()):
+                return name
+    tokens = re.findall(r'[A-Z][a-z]+(?:\s+[A-Z][a-z]+)*', text)
+    if tokens:
+        return tokens[0]
+    return text.split()[0]
+
+def normalize_name(s):
+    return re.sub(r'[^a-z]', '', s.lower())
+
+def find_wikipedia_list_page(role, country, serp_api_key):
+    query = f'List of {role} of {country} site:en.wikipedia.org'
+    try:
+        params = {"engine":"google", "q": query, "api_key": st.secrets["SERPAPI_KEY"], "num": 1}
+        search = GoogleSearch(params)
+        res = search.get_dict()
+        organic = res.get("organic_results") or []
+        if organic:
+            return organic[0].get("link")
+    except Exception:
+        pass
+    cand = f"https://en.wikipedia.org/wiki/List_of_{role.replace(' ','_')}_of_{country.replace(' ','_')}"
+    return cand
+
+def parse_wikipedia_list(url):
+    try:
+        r = requests.get(url, timeout=8, headers={"User-Agent":"Mozilla/5.0"})
+        if r.status_code != 200:
+            return []
+        soup = BeautifulSoup(r.text, 'html.parser')
+        names = []
+        tables = soup.find_all("table", {"class": ["wikitable", "sortable"]})
+        for table in tables:
+            for tr in table.find_all("tr"):
+                tds = tr.find_all(["td", "th"])
+                if not tds:
+                    continue
+                textcells = [td.get_text(separator=" ").strip() for td in tds if td.get_text(strip=True)]
+                for cell in textcells[:2]:
+                    if re.search(r'\b(19|20)\d{2}\b', cell) and len(cell) < 30:
+                        continue
+                    if len(cell) > 1 and re.search(r'[A-Za-z]', cell):
+                        cleaned = re.sub(r'\[.*?\]|\(.*?\)', '', cell).strip()
+                        cand = re.split(r'\n|,|;|-', cleaned)[0].strip()
+                        if len(cand) > 1 and not re.search(r'\b(year|term|born)\b', cand, re.I):
+                            names.append(cand)
+                            break
+        if not names:
+            for li in soup.find_all('li'):
+                text = li.get_text().strip()
+                if len(text) > 3 and re.search(r'\b[A-Z][a-z]+', text):
+                    if re.search(r'\b(19|20)\d{2}\b', text) or re.search(r'\bPrime Minister\b', text, re.I):
+                        cleaned = re.sub(r'\[.*?\]|\(.*?\)', '', text).strip()
+                        names.append(cleaned.split('—')[0].split('-')[0].strip())
+        normalized = []
+        for n in names:
+            nn = re.sub(r'\s+', ' ', n).strip()
+            if nn and nn not in normalized:
+                normalized.append(nn)
+        return normalized
+    except Exception:
+        return []
+
+def match_person_in_list(person_candidate, names_list):
+    pc = normalize_name(person_candidate)
+    for idx, full in enumerate(names_list):
+        if not full:
+            continue
+        fn = normalize_name(full)
+        if pc and (pc in fn or fn in pc):
+            return idx+1, full
+    tokens = person_candidate.lower().split()
+    for idx, full in enumerate(names_list):
+        fn = full.lower()
+        if all(any(tok in part for part in fn.split()) for tok in tokens if len(tok)>2):
+            return idx+1, full
+    return None, None
+
+def check_rank_claim_wikipedia(person, ordinal, role, country, serp_api_key):
+    wiki_url = find_wikipedia_list_page(role, country, serp_api_key)
+    names = parse_wikipedia_list(wiki_url)
+    if not names:
+        return {"decisive": False, "reason": "Could not retrieve list page or parse it.", "wiki_url": wiki_url}
+    rank, matched_name = match_person_in_list(person, names)
+    if rank is None:
+        return {"decisive": False, "reason": "Person not found in list parsed from " + wiki_url, "wiki_url": wiki_url, "names_sample": names[:6]}
+    else:
+        if rank == ordinal:
+            return {"decisive": True, "result": True, "rank": rank, "matched_name": matched_name, "wiki_url": wiki_url}
+        else:
+            return {"decisive": True, "result": False, "rank": rank, "matched_name": matched_name, "wiki_url": wiki_url}
+
+# ---------------- NLI & sentence helpers ----------------
+def nli_entailment_prob(premise, hypothesis):
+    inputs = nli_tok.encode_plus(premise, hypothesis, return_tensors="pt", truncation=True, max_length=512)
+    inputs = {k: v.to('cpu') for k, v in inputs.items()}
+    with torch.no_grad():
+        logits = nli_model(**inputs).logits
+        probs = F.softmax(logits, dim=1)[0]
+    # NLI labels for roberta-large-mnli are: 0=entailment, 1=neutral, 2=contradiction
+    return probs[0].item(), probs[1].item(), probs[2].item() # Entailment, Neutral, Contradiction
+
+def best_sentence_for_claim(snippet, claim):
+    import re
+    sents = re.split(r'(?<=[.!?])\s+', snippet) if snippet else []
+    if not sents:
+        return snippet or "", 0.0
+    sent_embs = embedder.encode(sents, convert_to_tensor=True)
+    claim_emb = embedder.encode(claim, convert_to_tensor=True)
+    sims = util.cos_sim(claim_emb, sent_embs)[0].cpu().numpy()
+    best_idx = int(sims.argmax())
+    return sents[best_idx], float(sims[best_idx])
+
+def domain_boost(domain):
+    trusted = ["bbc", "reuters", "theguardian", "nytimes", "indiatimes", "ndtv", "timesofindia", "cnn", "espn", "espncricinfo", "aljazeera"]
+    return 0.2 if any(t in domain for t in trusted) else 0.0
+
+def analyze_top_articles(normalized, claim, top_k):
+    sims, entails, neutral, contradicts, creds = [], [], [], [], []
+    for r in normalized[:top_k]:
+        text = (r.get("title","") + ". " + (r.get("snippet") or ""))
+        best_sent, best_sim = best_sentence_for_claim(r.get("snippet",""), claim)
+        # fallback semantic sim using whole text if best_sim==0
+        sem_sim = best_sim if best_sim>0 else float(util.cos_sim(
+            embedder.encode(claim, convert_to_tensor=True),
+            embedder.encode(text, convert_to_tensor=True)
+        )[0].item())
+        
+        try:
+            entail_p, neutral_p, contra_p = nli_entailment_prob(best_sent or text, claim)
+        except Exception:
+            entail_p, neutral_p, contra_p = 0.0, 0.0, 0.0
+        
+        # --- NLI Smart Filter (Fixes high contradiction on high sim matches) ---
+        if sem_sim > 0.80 and contra_p > 0.80 and entail_p < 0.10:
+             # Assume this is the correct news headline reporting the claim, not contradicting it.
+             entail_p = 0.80
+             contra_p = 0.05
+        # ----------------------------------------------------------------------
+        
+        domain = urlparse(r.get("link","")).netloc
+        cred = domain_boost(domain)
+        
+        sims.append(sem_sim)
+        entails.append(entail_p)
+        neutral.append(neutral_p)
+        contradicts.append(contra_p)
+        creds.append(cred)
+        
+        r["entail_p"] = entail_p
+        r["neutral_p"] = neutral_p
+        r["contra_p"] = contra_p
+        r["sem_sim"] = sem_sim
+        r["cred"] = cred
+        r["best_sent"] = best_sent 
+        
+    avg_sim = float(np.mean(sims)) if sims else 0.0
+    avg_ent = float(np.mean(entails)) if entails else 0.0
+    avg_neu = float(np.mean(neutral)) if neutral else 0.0
+    avg_con = float(np.mean(contradicts)) if contradicts else 0.0
+    avg_cred = float(np.mean(creds)) if creds else 0.0
+
+    # Calculate net support as (Entailment - Contradiction)
+    net_support = avg_ent - avg_con
+    
+    # DYNAMIC SCORING LOGIC
+    # SCORE 1: Support Score (Prioritizes credible logical support)
+    # This is the primary decision factor: Net Support * (1 + Credibility)
+    support_score = net_support * (1 + avg_cred)
+    
+    # SCORE 2: Final Score (Used for general ranking/transparency)
+    final_score = 0.50 * net_support + 0.30 * avg_sim + 0.20 * avg_cred
+    
+    metrics = {
+        "avg_ent": avg_ent, 
+        "avg_neu": avg_neu, 
+        "avg_con": avg_con, 
+        "avg_sim": avg_sim, 
+        "avg_cred": avg_cred, 
+        "net_support": net_support,
+        "support_score": support_score
+    }
+    return final_score, metrics, normalized[:top_k]
+
+# ---------------- Main UI inputs ----------------
+claim = st.text_area("Enter claim or news sentence:", height=140, placeholder="e.g. India defeats Pakistan in Asia Cup 2025")
+
+st.info(f"Using **{NUM_RESULTS}** recent news results (Last 24hrs) and analyzing top **{TOP_K_FOR_VERDICT}** matches (Full Power Mode).")
+
+if st.button("Verify Claim"):
+    if not claim.strip():
+        st.warning("Please enter a claim.")
+    else:
+        with st.spinner("Analysing... (this may take a few seconds)"):
+            
+            # 1) Rank-claim check (Wikipedia) if applicable
+            ordinal, role = find_ordinal_and_role(claim)
+            person_candidate = None
+            country = "India" if "india" in claim.lower() else "" 
+            if ordinal and role:
+                person_candidate = extract_person_candidate(claim)
+                m_country = re.search(r'\bof\s+([A-Za-z\s]+)', claim, flags=re.IGNORECASE)
+                if m_country:
+                    country = m_country.group(1).strip()
+                rank_check = check_rank_claim_wikipedia(person_candidate, ordinal, role, country or "India", st.secrets["SERPAPI_KEY"])
+                if rank_check.get("decisive"):
+                    if rank_check.get("result"):
+                        st.markdown("<h2 style='color:green;text-align:center'>✅ TRUE</h2>", unsafe_allow_html=True)
+                        st.write(f"Reason: Authoritative list ({rank_check.get('wiki_url')}) shows **{rank_check.get('matched_name')}** as the {ordinal}th {role} of {country or 'the country'}.")
+                    else:
+                        st.markdown("<h2 style='color:red;text-align:center'>🚨 FAKE</h2>", unsafe_allow_html=True)
+                        st.write(f"Reason: Authoritative list ({rank_check.get('wiki_url')}) shows **{rank_check.get('matched_name')}** as the {rank_check.get('rank')}th {role}, not the {ordinal}th.")
+                    st.write("Source (for verification):", rank_check.get("wiki_url"))
+                    st.stop()  # done
+
+            # 2) SerpAPI fetch (Filtering results to last 24hrs using tbs=qdr:d1)
+            try:
+                # Using tbs=qdr:d1 to filter results to the last 24 hours for better relevance
+                params = {"engine":"google", "q": claim, "tbm":"nws", "tbs":"qdr:d1", "num": NUM_RESULTS, "api_key": st.secrets["SERPAPI_KEY"]}
+                search = GoogleSearch(params)
+                data = search.get_dict()
+                results = data.get("news_results") or data.get("organic_results") or []
+            except Exception as e:
+                st.error("Search failed: " + str(e))
+                results = []
+
+            if not results:
+                st.markdown("<h2 style='color:red;text-align:center'>🚨 FAKE</h2>", unsafe_allow_html=True)
+                st.write("Reason: No relevant **recent** news results returned by the live search API. The claim is unconfirmed or outdated.")
+            else:
+                normalized = []
+                for r in results:
+                    title = r.get("title") or r.get("title_raw") or r.get("title_original") or ""
+                    snippet = r.get("snippet") or r.get("snippet_highlighted") or r.get("excerpt") or ""
+                    link = r.get("link") or r.get("source", {}).get("url") or r.get("source_link") or ""
+                    normalized.append({"title": title, "snippet": snippet, "link": link})
+
+                # compute decision via new intelligence module
+                final_score, metrics, analyzed = analyze_top_articles(normalized, claim, top_k=TOP_K_FOR_VERDICT)
+
+                # DYNAMIC VERDICT LOGIC: (TRUE / FAKE / INSUFFICIENT DATA)
+                
+                # Condition for TRUE: High credibility-weighted support AND good relevance.
+                if metrics["support_score"] >= 0.15 and metrics["avg_sim"] >= 0.50:
+                    st.markdown("<h2 style='color:green;text-align:center'>✅ TRUE</h2>", unsafe_allow_html=True)
+                    st.write("Reason: **Strong logical support from credible sources** found, confirming the claim's relevance.")
+                    verdict_msg = "TRUE"
+                
+                # Condition for INSUFFICIENT DATA: Not enough support (low support_score) and low relevance, but high neutrality (no strong contradiction found).
+                elif metrics["avg_sim"] < 0.50 and metrics["avg_neu"] > 0.60:
+                    st.markdown("<h2 style='color:orange;text-align:center'>⚠️ INSUFFICIENT DATA</h2>", unsafe_allow_html=True)
+                    st.write("Reason: Low semantic relevance and high neutral logical probability across sources. The claim is either too vague, futuristic, or lacks sufficient recent confirmation.")
+                    verdict_msg = "INSUFFICIENT DATA"
+
+                # Default to FAKE: Insufficient support or strong contradiction present.
+                else:
+                    st.markdown("<h2 style='color:red;text-align:center'>🚨 FAKE</h2>", unsafe_allow_html=True)
+                    st.write("Reason: Insufficient combined credibility and logical support, or strong refutation present. The claim is likely refuted, outdated, or lacks reliable confirmation.")
+                    verdict_msg = "FAKE"
+
+
+                st.write(f"Details — Support Score (Credibility Weighted): {metrics['support_score']:.2f}, avg semantic sim: {metrics['avg_sim']:.2f}, net support (E-C): {metrics['net_support']:.2f}")
+
+
+                # show short synthesized reason
+                if verdict_msg == "TRUE":
+                    ex = []
+                    for r in analyzed[:3]:
+                        if r.get("sem_sim", 0.0) > 0.4 and r.get("entail_p", 0.0) > r.get("contra_p", 0.0):
+                            ex.append(textwrap.shorten(r.get("best_sent") or r.get("snippet",""), width=160, placeholder="..."))
+                    if ex:
+                        st.info("Example supporting excerpts: " + " | ".join(ex))
+                elif verdict_msg == "FAKE":
+                    best = analyzed[0] if analyzed else None
+                    if best and best.get("best_sent"):
+                        st.info("Closest (but weak) excerpt: " + textwrap.shorten(best.get("best_sent") or best.get("snippet",""), width=220, placeholder="..."))
+
+                # transparency
+                with st.expander("Show analyzed top sources and scores"):
+                    for idx, r in enumerate(analyzed):
+                        st.markdown(f"**{idx+1}. {r.get('title') or r.get('link','(no title)')}**")
+                        st.write(f"- Domain: {domain_from_url(r.get('link',''))}")
+                        st.write(f"- Semantic similarity (sentence-level): {pretty_pct(r.get('sem_sim',0.0))}")
+                        st.write(f"- **Net Support (Entail-Contra)**: {r.get('entail_p',0.0) - r.get('contra_p',0.0):.2f}")
+                        st.write(f"  (E: {pretty_pct(r.get('entail_p',0.0))} | N: {pretty_pct(r.get('neutral_p',0.0))} | C: {pretty_pct(r.get('contra_p',0.0))})")
+                        st.write(f"- Credibility boost: {r.get('cred',0.0):.2f}")
+                        st.write(f"- Link: {r.get('link')}")
+                        st.markdown("---")
+
+
+# Footer
+st.markdown("---")
+st.caption("Project: NLP-driven Fact-Checking System. Use responsibly.")