Spaces:
Running
Running
| import gradio as gr | |
| import re | |
| from datatrove.pipeline.readers import ParquetReader | |
| from default_wiki_pipeline import _parse_and_clean_wikicode, mwparserfromhell | |
| lang_list = ['ab', 'ace', 'ady', 'af', 'als', 'alt', 'ami', 'am', 'ang', 'anp', 'an', 'arc', 'ar', 'ary', 'arz', 'ast', 'as', 'atj', 'avk', 'av', 'awa', 'ay', 'azb', 'az', 'ban', 'bar', 'bat_smg', 'ba', 'bbc', 'bcl', 'be', 'bg', 'bh', 'bi', 'bjn', 'blk', 'bm', 'bn', 'bo', 'bpy', 'br', 'bs', 'bug', 'bxr', 'ca', 'cbk_zam', 'cdo', 'ceb', 'ce', 'chr', 'ch', 'chy', 'ckb', 'co', 'crh', 'cr', 'csb', 'cs', 'cu', 'cv', 'cy', 'dag', 'da', 'de', 'dga', 'din', 'diq', 'dsb', 'dty', 'dv', 'dz', 'ee', 'el', 'eml', 'en', 'eo', 'es', 'et', 'eu', 'ext', 'fat', 'fa', 'ff', 'fiu_vro', 'fi', 'fj', 'fon', 'fo', 'frp', 'frr', 'fr', 'fur', 'fy', 'gag', 'gan', 'ga', 'gcr', 'gd', 'glk', 'gl', 'gn', 'gom', 'gor', 'got', 'gpe', 'guc', 'gur', 'gu', 'guw', 'gv', 'hak', 'ha', 'haw', 'he', 'hif', 'hi', 'hr', 'hsb', 'ht', 'hu', 'hy', 'hyw', 'ia', 'id', 'ie', 'ig', 'ik', 'ilo', 'inh', 'io', 'is', 'it', 'iu', 'jam', 'ja', 'jbo', 'jv', 'kaa', 'kab', 'ka', 'kbd', 'kbp', 'kcg', 'kg', 'ki', 'kk', 'kl', 'km', 'kn', 'koi', 'ko', 'krc', 'ksh', 'ks', 'ku', 'kv', 'kw', 'ky', 'lad', 'la', 'lbe', 'lb', 'lez', 'lfn', 'lg', 'lij', 'li', 'lld', 'lmo', 'ln', 'lo', 'ltg', 'lt', 'lv', 'mad', 'mai', 'map_bms', 'mdf', 'mg', 'mhr', 'min', 'mi', 'mk', 'ml', 'mni', 'mn', 'mnw', 'mrj', 'mr', 'ms', 'mt', 'mwl', 'myv', 'my', 'mzn', 'nah', 'nap', 'nds_nl', 'nds', 'ne', 'new', 'nia', 'nl', 'nn', 'nov', 'no', 'nqo', 'nrm', 'nso', 'nv', 'ny', 'oc', 'olo', 'om', 'or', 'os', 'pag', 'pam', 'pap', 'pa', 'pcd', 'pcm', 'pdc', 'pfl', 'pih', 'pi', 'pl', 'pms', 'pnb', 'pnt', 'ps', 'pt', 'pwn', 'qu', 'rm', 'rmy', 'rn', 'roa_rup', 'roa_tara', 'ro', 'rue', 'ru', 'rw', 'sah', 'sat', 'sa', 'scn', 'sco', 'sc', 'sd', 'se', 'sg', 'shi', 'shn', 'sh', 'simple', 'si', 'skr', 'sk', 'sl', 'smn', 'sm', 'sn', 'so', 'sq', 'srn', 'sr', 'ss', 'stq', 'st', 'su', 'sv', 'sw', 'szl', 'szy', 'ta', 'tay', 'tcy', 'tet', 'te', 'tg', 'th', 'ti', 'tk', 'tl', 'tly', 'tn', 'to', 'tpi', 'trv', 'tr', 'ts', 'tt', 'tum', 'tw', 'tyv', 'ty', 'udm', 'ug', 'uk', 'ur', 'uz', 'vec', 'vep', 've', 'vi', 'vls', 'vo', 'war', 'wa', 'wo', 'wuu', 'xal', 'xh', 'xmf', 'yi', 'yo', 'za', 'zea', 'zgh', 'zh_classical', 'zh_min_nan', 'zh_yue', 'zh', 'zu'] | |
| def _build_header_markdown(doc) -> str: | |
| meta = doc.metadata or {} | |
| title = meta.get("title") or "" | |
| page_id = meta.get("page_id") or meta.get("id") or "" | |
| wikidata_id = meta.get("wikidata_id") or "" | |
| url = meta.get("url") or "" | |
| parts = [] | |
| if title: | |
| parts.append(f"**Title**: {title}") | |
| if page_id: | |
| parts.append(f"**Page ID**: {page_id}") | |
| if wikidata_id: | |
| parts.append(f"**Wikidata ID**: {wikidata_id}") | |
| header = " | ".join(parts) | |
| if url: | |
| header += f"\n\n[{url}]({url})" | |
| return header | |
| def has_markdown_table(md_text: str) -> bool: | |
| return bool(re.search(r"(?m)^\s*\|.+\|\s*\n\s*\|?\s*:?-{3,}.*$", md_text or "")) | |
| def has_code_fence(md_text: str) -> bool: | |
| return "```" in (md_text or "") | |
| def matches_prefilters(doc, require_has_math: bool | None, require_has_infobox: bool | None) -> bool: | |
| meta = doc.metadata or {} | |
| if require_has_math and not bool(meta.get("has_math")): | |
| return False | |
| if require_has_infobox and meta.get("infoboxes", "[]") == "[]": | |
| return False | |
| return True | |
| def postfilters_ok(md_text: str, require_has_table: bool | None, require_has_code: bool | None) -> bool: | |
| if require_has_table and not has_markdown_table(md_text): | |
| return False | |
| if require_has_code and not has_code_fence(md_text): | |
| return False | |
| return True | |
| def format_for_markdown(text: str) -> str: | |
| import re | |
| # Replace '\n' not preceded by '|' with '\n\n' | |
| return re.sub(r'(?<!\|)\n', '\n\n', text) | |
| def find_next_valid(docs_cache, reader_iter, start_idx: int, require_has_math: bool | None, require_has_infobox: bool | None, require_has_table: bool | None, require_has_code: bool | None): | |
| # Scan cache first (forward from start_idx) | |
| i = max(-1, start_idx) | |
| while i + 1 < len(docs_cache): | |
| i += 1 | |
| if not matches_prefilters(docs_cache[i], require_has_math, require_has_infobox): | |
| continue | |
| left, left_meta, md, info, right, header = render_idx(docs_cache, i) | |
| if postfilters_ok(left, require_has_table, require_has_code): | |
| return i, docs_cache, reader_iter, left, md, left_meta, header, md, info, right | |
| # Stream until found or exhausted | |
| while True: | |
| prev_len = len(docs_cache) | |
| docs_cache, reader_iter = _ensure_until_index(docs_cache, reader_iter, prev_len) | |
| if len(docs_cache) == prev_len: | |
| break | |
| if not matches_prefilters(docs_cache[-1], require_has_math, require_has_infobox): | |
| continue | |
| left, left_meta, md, info, right, header = render_idx(docs_cache, len(docs_cache) - 1) | |
| if postfilters_ok(left, require_has_table, require_has_code): | |
| idx = len(docs_cache) - 1 | |
| return idx, docs_cache, reader_iter, left, md, left_meta, header, md, info, right | |
| return -1, docs_cache, reader_iter, "No documents match filters.", "", {}, "", "", [], render_iframe("") | |
| def render_iframe(url: str, height: int = 800) -> str: | |
| safe_url = url or "about:blank" | |
| return ( | |
| f'<iframe src="{safe_url}" ' | |
| f'style="width:100%; height:{height}px; border:0;" loading="lazy"></iframe>' | |
| ) | |
| def _safe_url_from_metadata(meta: dict) -> str: | |
| meta = meta or {} | |
| return meta.get("url") or "" | |
| def _extract_language(meta: dict) -> str: | |
| # Try common metadata fields for language code | |
| meta = meta or {} | |
| lang = meta.get("lang") or meta.get("language") | |
| if lang: | |
| return str(lang) | |
| wiki = meta.get("wiki") or meta.get("wikiname") or "" | |
| base = str(wiki).removesuffix("_namespace_0") if wiki else "" | |
| if base.endswith("wiki"): | |
| return base[:-4] | |
| return base or "en" | |
| def _ensure_until_index(docs_cache, reader_iter, target_idx: int): | |
| if reader_iter is None: | |
| return docs_cache, reader_iter | |
| while len(docs_cache) <= target_idx: | |
| try: | |
| nxt = next(reader_iter) | |
| except StopIteration: | |
| break | |
| docs_cache.append(nxt) | |
| return docs_cache, reader_iter | |
| def on_select_language(lang: str, require_has_math: bool, require_has_infobox: bool, require_has_table: bool, require_has_code: bool): | |
| """Load documents for the selected language from HF Parquet and display.""" | |
| language = (lang or "").strip() | |
| if not language: | |
| return (-1, [], None, "Select a language.", {}, "", [], render_iframe("")) | |
| try: | |
| path = f"hf://datasets/HuggingFaceFW/finewiki/data/{language}wiki" | |
| reader_iter = ParquetReader(path)() | |
| except Exception as e: | |
| return (-1, [], None, f"Failed to read: {e}", {}, "", [], render_iframe("")) | |
| docs_cache = [] | |
| docs_cache, reader_iter = _ensure_until_index(docs_cache, reader_iter, 0) | |
| if not docs_cache: | |
| return (-1, [], reader_iter, "No documents found.", {}, "", [], render_iframe("")) | |
| # Find first doc matching pre- and post-filters | |
| idx, docs_cache, reader_iter, left, md, left_meta, header, right_md, info, right = find_next_valid( | |
| docs_cache, reader_iter, -1, require_has_math, require_has_infobox, require_has_table, require_has_code | |
| ) | |
| if idx == -1: | |
| return (-1, docs_cache, reader_iter, "No documents match filters.", "", {}, "", "", [], render_iframe("")) | |
| return (idx, docs_cache, reader_iter, left, format_for_markdown(left), left_meta, header, right_md, info, right) | |
| def show_doc(doc): | |
| left = getattr(doc, "text", "") | |
| meta = getattr(doc, "metadata", None) or {} | |
| # Clean markdown using default_wiki_pipeline helper | |
| md_text = meta.get("wikitext") | |
| md_clean = _parse_and_clean_wikicode(md_text, parser=mwparserfromhell, language=_extract_language(meta)) | |
| info = meta.get("infoboxes", []) | |
| right = render_iframe(_safe_url_from_metadata(meta)) | |
| header = _build_header_markdown(doc) | |
| return left, meta, md_clean, info, right, header | |
| def render_idx(docs, idx: int): | |
| if not docs: | |
| return "No documents.", {}, "", [], render_iframe(""), "" | |
| idx = max(0, min(idx, len(docs) - 1)) | |
| doc = docs[idx] | |
| left, left_meta, md, info, right, header = show_doc(doc) | |
| return left, left_meta, md, info, right, header | |
| def on_prev(docs_cache, idx: int, reader_iter, require_has_math: bool, require_has_infobox: bool, require_has_table: bool, require_has_code: bool): | |
| if not docs_cache: | |
| # Try to ensure at least first doc is loaded | |
| docs_cache, reader_iter = _ensure_until_index(docs_cache, reader_iter, 0) | |
| if not docs_cache: | |
| return idx, docs_cache, reader_iter, "No documents.", {}, "", [], render_iframe("") | |
| new_idx = max(0, idx - 1) | |
| # Apply prefilters going backwards by scanning from start to new_idx; evaluate postfilters on candidate | |
| filtered_idx = -1 | |
| for i in range(new_idx, -1, -1): | |
| if not matches_prefilters(docs_cache[i], require_has_math, require_has_infobox): | |
| continue | |
| left, left_meta, md, info, right, header = render_idx(docs_cache, i) | |
| if postfilters_ok(md, require_has_table, require_has_code): | |
| filtered_idx = i | |
| return filtered_idx, docs_cache, reader_iter, left, format_for_markdown(left), left_meta, header, md, info, right | |
| return idx, docs_cache, reader_iter, "No documents match filters.", "", {}, "", "", [], render_iframe("") | |
| def on_next(docs_cache, idx: int, reader_iter, require_has_math: bool, require_has_infobox: bool, require_has_table: bool, require_has_code: bool): | |
| target_idx = idx + 1 if idx >= 0 else 0 | |
| docs_cache, reader_iter = _ensure_until_index(docs_cache, reader_iter, target_idx) | |
| if not docs_cache: | |
| return idx, docs_cache, reader_iter, "No documents.", {}, "", [], render_iframe("") | |
| # Apply filters forward using new finder | |
| new_idx, docs_cache, reader_iter, left, md, left_meta, header, right_md, info, right = find_next_valid( | |
| docs_cache, reader_iter, idx, require_has_math, require_has_infobox, require_has_table, require_has_code | |
| ) | |
| if new_idx == -1: | |
| return idx, docs_cache, reader_iter, "No documents match filters.", "", {}, "", "", [], render_iframe("") | |
| return new_idx, docs_cache, reader_iter, left, format_for_markdown(left), left_meta, header, right_md, info, right | |
| with gr.Blocks() as demo: | |
| idx_state = gr.State(value=-1, time_to_live=900) | |
| docs_state = gr.State(value=[], time_to_live=900) | |
| iter_state = gr.State(value=None, time_to_live=900) | |
| # Full-width controls row for navigation | |
| with gr.Row(): | |
| with gr.Column(): | |
| with gr.Row(): | |
| header_md = gr.Markdown() | |
| with gr.Row(): | |
| prev_btn = gr.Button("Previous") | |
| next_btn = gr.Button("Next") | |
| with gr.Column(): | |
| with gr.Row(): | |
| with gr.Column(scale=1): | |
| language_select = gr.Dropdown(choices=lang_list, value="en", label="Language") | |
| with gr.Column(scale=2): | |
| with gr.Row(): | |
| require_has_math = gr.Checkbox(label="Has math", value=False) | |
| require_has_infobox = gr.Checkbox(label="Has infobox", value=False) | |
| require_has_table = gr.Checkbox(label="Has table", value=False) | |
| require_has_code = gr.Checkbox(label="Has pre/code", value=False) | |
| with gr.Row(): | |
| with gr.Column(): | |
| with gr.Tab("FineWiki markdown"): | |
| left_text_md = gr.Markdown(label="FineWiki markdown") | |
| with gr.Tab("FineWiki raw"): | |
| left_text_raw = gr.Textbox(label="FineWiki extraction", lines=30, elem_id="left_text_box") | |
| with gr.Tab("FineWiki metadata"): | |
| left_meta = gr.JSON(label="Metadata") | |
| with gr.Tab("FineWiki infoboxes"): | |
| right_infoboxes = gr.JSON(label="Infoboxes") | |
| with gr.Row(): | |
| prev_btn2 = gr.Button("Previous") | |
| next_btn2 = gr.Button("Next") | |
| with gr.Column(): | |
| with gr.Tab("Preview"): | |
| right_iframe = gr.HTML(label="Original Page") | |
| with gr.Tab("wikimedia/wikipedia"): | |
| right_markdown = gr.Textbox(label="wikimedia/wikipedia extraction", lines=30, elem_id="right_markdown_box") | |
| language_select.change(on_select_language, inputs=[language_select, require_has_math, require_has_infobox, require_has_table, require_has_code], outputs=[idx_state, docs_state, iter_state, left_text_raw, left_text_md, left_meta, header_md, right_markdown, right_infoboxes, right_iframe], concurrency_limit=1) | |
| demo.load(on_select_language, inputs=[language_select, require_has_math, require_has_infobox, require_has_table, require_has_code], outputs=[idx_state, docs_state, iter_state, left_text_raw, left_text_md, left_meta, header_md, right_markdown, right_infoboxes, right_iframe], concurrency_limit=1) | |
| # find_btn.click(on_find, inputs=[docs_state, idx_state, iter_state, id_input, require_has_math, require_has_infobox], outputs=[idx_state, docs_state, iter_state, left_text, left_meta, header_md, right_markdown, right_infoboxes, right_iframe], concurrency_limit=1) | |
| # Visibility toggles driven directly by checkbox changes | |
| prev_btn.click(on_prev, inputs=[docs_state, idx_state, iter_state, require_has_math, require_has_infobox, require_has_table, require_has_code], outputs=[idx_state, docs_state, iter_state, left_text_raw, left_text_md, left_meta, header_md, right_markdown, right_infoboxes, right_iframe], concurrency_limit=1) | |
| next_btn.click(on_next, inputs=[docs_state, idx_state, iter_state, require_has_math, require_has_infobox, require_has_table, require_has_code], outputs=[idx_state, docs_state, iter_state, left_text_raw, left_text_md, left_meta, header_md, right_markdown, right_infoboxes, right_iframe], concurrency_limit=1) | |
| prev_btn2.click(on_prev, inputs=[docs_state, idx_state, iter_state, require_has_math, require_has_infobox, require_has_table, require_has_code], outputs=[idx_state, docs_state, iter_state, left_text_raw, left_text_md, left_meta, header_md, right_markdown, right_infoboxes, right_iframe], concurrency_limit=1) | |
| next_btn2.click(on_next, inputs=[docs_state, idx_state, iter_state, require_has_math, require_has_infobox, require_has_table, require_has_code], outputs=[idx_state, docs_state, iter_state, left_text_raw, left_text_md, left_meta, header_md, right_markdown, right_infoboxes, right_iframe], concurrency_limit=1) | |
| # Enable global queue to coordinate concurrent requests safely | |
| demo.queue(default_concurrency_limit=1, max_size=128) | |
| if __name__ == "__main__": | |
| demo.launch(server_name="0.0.0.0", state_session_capacity=5000) | |