add app

README.md

@@ -1,8 +1,8 @@
 ---
-title: Finewiki Viewer
-emoji: 🏢
-colorFrom: blue
-colorTo: green
+title: FineWiki Viewer
+emoji: 🌐
+colorFrom: white
+colorTo: black
 sdk: gradio
 sdk_version: 5.49.1
 app_file: app.py

app.py

@@ -0,0 +1,293 @@
+# import os
+# _LOCAL_TMP = "/fsx/guilherme/tmp"
+# try:
+#     os.makedirs(_LOCAL_TMP, exist_ok=True)
+#     os.environ.setdefault("TMPDIR", _LOCAL_TMP)
+#     os.environ.setdefault("TEMP", _LOCAL_TMP)
+#     os.environ.setdefault("TMP", _LOCAL_TMP)
+#     _GRADIO_TMP = os.path.join(_LOCAL_TMP, "gradio")
+#     os.makedirs(_GRADIO_TMP, exist_ok=True)
+#     os.environ.setdefault("GRADIO_TEMP_DIR", _GRADIO_TMP)
+# except Exception:
+#     pass
+
+import gradio as gr
+from datatrove.pipeline.readers import ParquetReader
+from default_wiki_pipeline import _parse_and_clean_wikicode, mwparserfromhell
+
+lang_list = ['ab', 'ace', 'ady', 'af', 'als', 'alt', 'ami', 'am', 'ang', 'anp', 'an', 'arc', 'ar', 'ary', 'arz', 'ast', 'as', 'atj', 'avk', 'av', 'awa', 'ay', 'azb', 'az', 'ban', 'bar', 'bat_smg', 'ba', 'bbc', 'bcl', 'be', 'bg', 'bh', 'bi', 'bjn', 'blk', 'bm', 'bn', 'bo', 'bpy', 'br', 'bs', 'bug', 'bxr', 'ca', 'cbk_zam', 'cdo', 'ceb', 'ce', 'chr', 'ch', 'chy', 'ckb', 'co', 'crh', 'cr', 'csb', 'cs', 'cu', 'cv', 'cy', 'dag', 'da', 'de', 'dga', 'din', 'diq', 'dsb', 'dty', 'dv', 'dz', 'ee', 'el', 'eml', 'en', 'eo', 'es', 'et', 'eu', 'ext', 'fat', 'fa', 'ff', 'fiu_vro', 'fi', 'fj', 'fon', 'fo', 'frp', 'frr', 'fr', 'fur', 'fy', 'gag', 'gan', 'ga', 'gcr', 'gd', 'glk', 'gl', 'gn', 'gom', 'gor', 'got', 'gpe', 'guc', 'gur', 'gu', 'guw', 'gv', 'hak', 'ha', 'haw', 'he', 'hif', 'hi', 'hr', 'hsb', 'ht', 'hu', 'hy', 'hyw', 'ia', 'id', 'ie', 'ig', 'ik', 'ilo', 'inh', 'io', 'is', 'it', 'iu', 'jam', 'ja', 'jbo', 'jv', 'kaa', 'kab', 'ka', 'kbd', 'kbp', 'kcg', 'kg', 'ki', 'kk', 'kl', 'km', 'kn', 'koi', 'ko', 'krc', 'ksh', 'ks', 'ku', 'kv', 'kw', 'ky', 'lad', 'la', 'lbe', 'lb', 'lez', 'lfn', 'lg', 'lij', 'li', 'lld', 'lmo', 'ln', 'lo', 'ltg', 'lt', 'lv', 'mad', 'mai', 'map_bms', 'mdf', 'mg', 'mhr', 'min', 'mi', 'mk', 'ml', 'mni', 'mn', 'mnw', 'mrj', 'mr', 'ms', 'mt', 'mwl', 'myv', 'my', 'mzn', 'nah', 'nap', 'nds_nl', 'nds', 'ne', 'new', 'nia', 'nl', 'nn', 'nov', 'no', 'nqo', 'nrm', 'nso', 'nv', 'ny', 'oc', 'olo', 'om', 'or', 'os', 'pag', 'pam', 'pap', 'pa', 'pcd', 'pcm', 'pdc', 'pfl', 'pih', 'pi', 'pl', 'pms', 'pnb', 'pnt', 'ps', 'pt', 'pwn', 'qu', 'rm', 'rmy', 'rn', 'roa_rup', 'roa_tara', 'ro', 'rue', 'ru', 'rw', 'sah', 'sat', 'sa', 'scn', 'sco', 'sc', 'sd', 'se', 'sg', 'shi', 'shn', 'sh', 'simple', 'si', 'skr', 'sk', 'sl', 'smn', 'sm', 'sn', 'so', 'sq', 'srn', 'sr', 'ss', 'stq', 'st', 'su', 'sv', 'sw', 'szl', 'szy', 'ta', 'tay', 'tcy', 'tet', 'te', 'tg', 'th', 'ti', 'tk', 'tl', 'tly', 'tn', 'to', 'tpi', 'trv', 'tr', 'ts', 'tt', 'tum', 'tw', 'tyv', 'ty', 'udm', 'ug', 'uk', 'ur', 'uz', 'vec', 'vep', 've', 'vi', 'vls', 'vo', 'war', 'wa', 'wo', 'wuu', 'xal', 'xh', 'xmf', 'yi', 'yo', 'za', 'zea', 'zgh', 'zh_classical', 'zh_min_nan', 'zh_yue', 'zh', 'zu']
+
+
+def _build_header_markdown(doc) -> str:
+    meta = doc.metadata or {}
+    title = meta.get("title") or ""
+    page_id = meta.get("page_id") or meta.get("id") or ""
+    wikidata_id = meta.get("wikidata_id") or ""
+    url = meta.get("url") or ""
+    parts = []
+    if title:
+        parts.append(f"**Title**: {title}")
+    if page_id:
+        parts.append(f"**Page ID**: {page_id}")
+    if wikidata_id:
+        parts.append(f"**Wikidata ID**: {wikidata_id}")
+    header = "  |  ".join(parts)
+    if url:
+        header += f"\n[{url}]({url})"
+    return header
+def matches_filters(doc, require_has_math: bool | None, require_has_infobox: bool | None) -> bool:
+    meta = doc.metadata or {}
+    if require_has_math and not bool(meta.get("has_math")):
+        return False
+    if require_has_infobox and not meta.get("infoboxes"):
+        return False
+    return True
+
+
+def find_next_matching_from(docs_cache, reader_iter, start_idx: int, require_has_math: bool | None, require_has_infobox: bool | None):
+    # Scan cache first
+    i = max(-1, start_idx)
+    while i + 1 < len(docs_cache):
+        i += 1
+        if matches_filters(docs_cache[i], require_has_math, require_has_infobox):
+            return i, docs_cache, reader_iter
+    # Stream until found or exhausted
+    while True:
+        prev_len = len(docs_cache)
+        docs_cache, reader_iter = _ensure_until_index(docs_cache, reader_iter, prev_len)
+        if len(docs_cache) == prev_len:
+            break
+        if matches_filters(docs_cache[-1], require_has_math, require_has_infobox):
+            return len(docs_cache) - 1, docs_cache, reader_iter
+    return -1, docs_cache, reader_iter
+
+def render_iframe(url: str, height: int = 800) -> str:
+    safe_url = url or "about:blank"
+    return (
+        f'<iframe src="{safe_url}" '
+        f'style="width:100%; height:{height}px; border:0;" loading="lazy"></iframe>'
+    )
+
+
+def _safe_url_from_metadata(meta: dict) -> str:
+    meta = meta or {}
+    return meta.get("url") or ""
+
+
+def _extract_language(meta: dict) -> str:
+    # Try common metadata fields for language code
+    meta = meta or {}
+    lang = meta.get("lang") or meta.get("language")
+    if lang:
+        return str(lang)
+    wiki = meta.get("wiki") or meta.get("wikiname") or ""
+    base = str(wiki).removesuffix("_namespace_0") if wiki else ""
+    if base.endswith("wiki"):
+        return base[:-4]
+    return base or "en"
+
+
+def _ensure_until_index(docs_cache, reader_iter, target_idx: int):
+    if reader_iter is None:
+        return docs_cache, reader_iter
+    while len(docs_cache) <= target_idx:
+        try:
+            nxt = next(reader_iter)
+        except StopIteration:
+            break
+        docs_cache.append(nxt)
+    return docs_cache, reader_iter
+
+
+def on_select_language(lang: str, require_has_math: bool, require_has_infobox: bool):
+    """Load documents for the selected language from HF Parquet and display."""
+    language = (lang or "").strip()
+    if not language:
+        return (-1, [], None, "Select a language.", {}, "", [], render_iframe(""))
+    try:
+        path = f"hf://datasets/HuggingFaceFW/finewiki/data/{language}wiki"
+        reader_iter = ParquetReader(path)()
+    except Exception as e:
+        return (-1, [], None, f"Failed to read: {e}", {}, "", [], render_iframe(""))
+    docs_cache = []
+    docs_cache, reader_iter = _ensure_until_index(docs_cache, reader_iter, 0)
+    if not docs_cache:
+        return (-1, [], reader_iter, "No documents found.", {}, "", [], render_iframe(""))
+    # Find first doc matching filters (starting before 0)
+    idx, docs_cache, reader_iter = find_next_matching_from(docs_cache, reader_iter, -1, require_has_math, require_has_infobox)
+    if idx == -1:
+        return (-1, docs_cache, reader_iter, "No documents match filters.", {}, "", [], render_iframe(""))
+    left, left_meta, md, info, right, header = render_idx(docs_cache, idx)
+    return (idx, docs_cache, reader_iter, left, left_meta, header, md, info, right)
+
+
+def on_find(docs_cache, idx: int, reader_iter, id_query: str, require_has_math: bool, require_has_infobox: bool):
+    query = (id_query or "").strip()
+    if not docs_cache and reader_iter is None:
+        return -1, docs_cache, reader_iter, "No documents loaded.", {}, "", [], render_iframe("")
+    if not query:
+        docs_cache, reader_iter = _ensure_until_index(docs_cache, reader_iter, 0)
+        new_idx, docs_cache, reader_iter = find_next_matching_from(docs_cache, reader_iter, -1, require_has_math, require_has_infobox)
+        if new_idx == -1:
+            return (-1, docs_cache, reader_iter, "No documents match filters.", {}, "", [], render_iframe(""))
+        left, left_meta, md, info, right, header = render_idx(docs_cache, new_idx)
+        return new_idx, docs_cache, reader_iter, left, left_meta, header, md, info, right
+    # Exact match in cache
+    for i, doc in enumerate(docs_cache):
+        meta = (getattr(doc, "metadata", None) or {})
+        doc_id = (getattr(doc, "id", None) or "")
+        url = meta.get("url") or ""
+        if doc_id == query or meta.get("wikidata_id") == query or url == query:
+            left, left_meta, md, info, right, header = render_idx(docs_cache, i)
+            if matches_filters(doc, require_has_math, require_has_infobox):
+                return i, docs_cache, reader_iter, left, left_meta, header, md, info, right
+            new_idx, docs_cache, reader_iter = find_next_matching_from(docs_cache, reader_iter, i, require_has_math, require_has_infobox)
+            if new_idx == -1:
+                return (-1, docs_cache, reader_iter, "No documents match filters.", {}, "", [], render_iframe(""))
+            left, left_meta, md, info, right, header = render_idx(docs_cache, new_idx)
+            return new_idx, docs_cache, reader_iter, left, left_meta, header, md, info, right
+    # Suffix match in cache
+    for i, doc in enumerate(docs_cache):
+        doc_id = (getattr(doc, "id", None) or "")
+        meta = (getattr(doc, "metadata", None) or {})
+        url = meta.get("url") or ""
+        if doc_id.endswith(f"/{query}") or url.endswith(query):
+            left, left_meta, md, info, right, header = render_idx(docs_cache, i)
+            if matches_filters(doc, require_has_math, require_has_infobox):
+                return i, docs_cache, reader_iter, left, left_meta, header, md, info, right
+            new_idx, docs_cache, reader_iter = find_next_matching_from(docs_cache, reader_iter, i, require_has_math, require_has_infobox)
+            if new_idx == -1:
+                return (-1, docs_cache, reader_iter, "No documents match filters.", {}, "", [], render_iframe(""))
+            left, left_meta, md, info, right, header = render_idx(docs_cache, new_idx)
+            return new_idx, docs_cache, reader_iter, left, left_meta, header, md, info, right
+    # Stream forward until found or exhausted
+    found_idx = None
+    while True:
+        prev_len = len(docs_cache)
+        docs_cache, reader_iter = _ensure_until_index(docs_cache, reader_iter, prev_len)
+        if len(docs_cache) == prev_len:
+            break
+        doc = docs_cache[-1]
+        meta = (getattr(doc, "metadata", None) or {})
+        doc_id = (getattr(doc, "id", None) or "")
+        url = meta.get("url") or ""
+        if doc_id == query or meta.get("wikidata_id") == query or url.endswith(query) or url == query or doc_id.endswith(f"/{query}"):
+            found_idx = len(docs_cache) - 1
+            break
+    if found_idx is not None:
+        new_idx, docs_cache, reader_iter = find_next_matching_from(docs_cache, reader_iter, found_idx - 1, require_has_math, require_has_infobox)
+        if new_idx == -1:
+            return (-1, docs_cache, reader_iter, "No documents match filters.", {}, "", [], render_iframe(""))
+        left, left_meta, md, info, right, header = render_idx(docs_cache, new_idx)
+        return new_idx, docs_cache, reader_iter, left, left_meta, header, md, info, right
+    target_idx = 0 if docs_cache else -1
+    if target_idx == -1:
+        return -1, docs_cache, reader_iter, "No documents found.", {}, "", [], render_iframe("")
+    new_idx, docs_cache, reader_iter = find_next_matching_from(docs_cache, reader_iter, target_idx - 1, require_has_math, require_has_infobox)
+    if new_idx == -1:
+        return (-1, docs_cache, reader_iter, "No documents match filters.", {}, "", [], render_iframe(""))
+    left, left_meta, md, info, right, header = render_idx(docs_cache, new_idx)
+    return new_idx, docs_cache, reader_iter, left, left_meta, header, md, info, right
+
+
+def show_doc(doc):
+    left = getattr(doc, "text", "")
+    meta = getattr(doc, "metadata", None) or {}
+    # Clean markdown using default_wiki_pipeline helper
+    md_text = meta.get("wikitext")
+    md_clean = _parse_and_clean_wikicode(md_text, parser=mwparserfromhell, language=_extract_language(meta))
+    info = meta.get("infoboxes", [])
+    right = render_iframe(_safe_url_from_metadata(meta))
+    header = _build_header_markdown(doc)
+    return left, meta, md_clean, info, right, header
+
+
+def render_idx(docs, idx: int):
+    if not docs:
+        return "No documents.", {}, "", [], render_iframe(""), ""
+    idx = max(0, min(idx, len(docs) - 1))
+    doc = docs[idx]
+    left, left_meta, md, info, right, header = show_doc(doc)
+    return left, left_meta, md, info, right, header
+
+
+def on_prev(docs_cache, idx: int, reader_iter, require_has_math: bool, require_has_infobox: bool):
+    if not docs_cache:
+        # Try to ensure at least first doc is loaded
+        docs_cache, reader_iter = _ensure_until_index(docs_cache, reader_iter, 0)
+        if not docs_cache:
+            return idx, docs_cache, reader_iter, "No documents.", {}, "", [], render_iframe("")
+    new_idx = max(0, idx - 1)
+    # Apply filters going backwards by scanning from start to new_idx
+    filtered_idx = new_idx
+    if new_idx >= 0:
+        for i in range(new_idx, -1, -1):
+            if matches_filters(docs_cache[i], require_has_math, require_has_infobox):
+                filtered_idx = i
+                break
+    left, left_meta, md, info, right, header = render_idx(docs_cache, filtered_idx)
+    return filtered_idx, docs_cache, reader_iter, left, left_meta, header, md, info, right
+
+
+def on_next(docs_cache, idx: int, reader_iter, require_has_math: bool, require_has_infobox: bool):
+    target_idx = idx + 1 if idx >= 0 else 0
+    docs_cache, reader_iter = _ensure_until_index(docs_cache, reader_iter, target_idx)
+    if not docs_cache:
+        return idx, docs_cache, reader_iter, "No documents.", {}, "", [], render_iframe("")
+    new_idx = min(len(docs_cache) - 1, target_idx)
+    # Apply filters forward
+    new_idx, docs_cache, reader_iter = find_next_matching_from(docs_cache, reader_iter, idx, require_has_math, require_has_infobox)
+    if new_idx == -1:
+        return idx, docs_cache, reader_iter, "No documents match filters.", {}, "", [], render_iframe("")
+    left, left_meta, md, info, right, header = render_idx(docs_cache, new_idx)
+    return new_idx, docs_cache, reader_iter, left, left_meta, header, md, info, right
+
+
+with gr.Blocks() as demo:
+    idx_state = gr.State(value=-1)
+    docs_state = gr.State(value=[])
+    iter_state = gr.State(value=None)
+
+    with gr.Row():
+        # Full-width controls row for navigation
+        with gr.Column():
+            with gr.Row():
+                language_select = gr.Dropdown(choices=lang_list, value="en", label="Language")
+            with gr.Row():
+                prev_btn = gr.Button("Previous")
+                next_btn = gr.Button("Next")
+            header_md = gr.Markdown()
+        with gr.Column():
+            with gr.Row():
+                require_has_math = gr.Checkbox(label="Has math", value=False)
+                require_has_infobox = gr.Checkbox(label="Has infobox", value=False)
+            with gr.Row():
+                id_input = gr.Textbox(label="Wikidata ID/URL/Page ID", placeholder="e.g., Q42 or https://... or 12345", lines=1)
+                find_btn = gr.Button("Find")
+            with gr.Row():
+                show_wiki = gr.Checkbox(label="Show wikimedia/wikipedia extraction", value=False)
+                show_preview = gr.Checkbox(label="Show preview", value=True)
+                show_infoboxes = gr.Checkbox(label="Show infoboxes", value=True)
+    with gr.Row():
+        with gr.Column():
+            left_text = gr.Textbox(label="FineWiki extractions", lines=30)
+            left_meta = gr.JSON(label="Metadata")
+        with gr.Column():
+            right_markdown = gr.Textbox(label="wikimedia/wikipedia extraction", lines=30)
+            right_iframe = gr.HTML(label="Original Page")
+            right_infoboxes = gr.JSON(label="Infoboxes")
+
+    language_select.change(on_select_language, inputs=[language_select, require_has_math, require_has_infobox], outputs=[idx_state, docs_state, iter_state, left_text, left_meta, header_md, right_markdown, right_infoboxes, right_iframe])
+    demo.load(on_select_language, inputs=[language_select, require_has_math, require_has_infobox], outputs=[idx_state, docs_state, iter_state, left_text, left_meta, header_md, right_markdown, right_infoboxes, right_iframe])
+    find_btn.click(on_find, inputs=[docs_state, idx_state, iter_state, id_input, require_has_math, require_has_infobox], outputs=[idx_state, docs_state, iter_state, left_text, left_meta, header_md, right_markdown, right_infoboxes, right_iframe])
+
+    # Visibility toggles driven directly by checkbox changes
+    show_wiki.change(lambda v: gr.update(visible=v), inputs=[show_wiki], outputs=[right_markdown])
+    show_preview.change(lambda v: gr.update(visible=v), inputs=[show_preview], outputs=[right_iframe])
+    show_infoboxes.change(lambda v: gr.update(visible=v), inputs=[show_infoboxes], outputs=[right_infoboxes])
+    prev_btn.click(on_prev, inputs=[docs_state, idx_state, iter_state, require_has_math, require_has_infobox], outputs=[idx_state, docs_state, iter_state, left_text, left_meta, header_md, right_markdown, right_infoboxes, right_iframe])
+    next_btn.click(on_next, inputs=[docs_state, idx_state, iter_state, require_has_math, require_has_infobox], outputs=[idx_state, docs_state, iter_state, left_text, left_meta, header_md, right_markdown, right_infoboxes, right_iframe])
+
+
+if __name__ == "__main__":
+    demo.launch(server_name="0.0.0.0", server_port=7641)

default_wiki_pipeline.py

@@ -0,0 +1,1273 @@
+# directly copied from https://huggingface.co/datasets/wikimedia/wikipedia/blob/script/wikipedia.py
+"""Wikipedia dataset containing cleaned articles of all languages."""
+
+
+import bz2
+import csv
+import io
+import itertools
+import json
+import re
+import xml.etree.ElementTree as etree
+from urllib.parse import quote
+
+import mwparserfromhell
+
+import datasets
+
+
+logger = datasets.logging.get_logger(__name__)
+
+
+_HOMEPAGE = "https://dumps.wikimedia.org"
+
+_CITATION = """\
+@ONLINE {wikidump,
+    author = {Wikimedia Foundation},
+    title  = {Wikimedia Downloads},
+    url    = {https://dumps.wikimedia.org}
+}
+"""
+
+_DESCRIPTION = """\
+Wikipedia dataset containing cleaned articles of all languages.
+The datasets are built from the Wikipedia dump
+(https://dumps.wikimedia.org/) with one split per language. Each example
+contains the content of one full Wikipedia article with cleaning to strip
+markdown and unwanted sections (references, etc.).
+"""
+
+_LICENSE = (
+    "This work is licensed under the Creative Commons Attribution-ShareAlike "
+    "3.0 Unported License. To view a copy of this license, visit "
+    "http://creativecommons.org/licenses/by-sa/3.0/ or send a letter to "
+    "Creative Commons, PO Box 1866, Mountain View, CA 94042, USA."
+)
+
+# Source: https://meta.wikimedia.org/wiki/List_of_Wikipedias
+# Number: 326 = 339 - 13 (retrieved: 2023-11-17)
+WIKIPEDIA_LANGUAGES = [
+    "ab",
+    "ace",
+    "ady",
+    "af",
+    "als",
+    "alt",
+    "am",
+    "ami",
+    "an",
+    "ang",
+    "anp",
+    "ar",
+    "arc",
+    "ary",
+    "arz",
+    "as",
+    "ast",
+    "atj",
+    "av",
+    "avk",
+    "awa",
+    "ay",
+    "az",
+    "azb",
+    "ba",
+    "ban",
+    "bar",
+    "bat-smg",
+    "bbc",
+    "bcl",
+    "be",
+    "be-tarask",
+    "bg",
+    "bh",
+    "bi",
+    "bjn",
+    "blk",
+    "bm",
+    "bn",
+    "bo",
+    "bpy",
+    "br",
+    "bs",
+    "bug",
+    "bxr",
+    "ca",
+    "cbk-zam",
+    "cdo",
+    "ce",
+    "ceb",
+    "ch",
+    "chr",
+    "chy",
+    "ckb",
+    "co",
+    "cr",
+    "crh",
+    "cs",
+    "csb",
+    "cu",
+    "cv",
+    "cy",
+    "da",
+    "dag",
+    "de",
+    "dga",
+    "din",
+    "diq",
+    "dsb",
+    "dty",
+    "dv",
+    "dz",
+    "ee",
+    "el",
+    "eml",
+    "en",
+    "eo",
+    "es",
+    "et",
+    "eu",
+    "ext",
+    "fa",
+    "fat",
+    "ff",
+    "fi",
+    "fiu-vro",
+    "fj",
+    "fo",
+    "fon",
+    "fr",
+    "frp",
+    "frr",
+    "fur",
+    "fy",
+    "ga",
+    "gag",
+    "gan",
+    "gcr",
+    "gd",
+    "gl",
+    "glk",
+    "gn",
+    "gom",
+    "gor",
+    "got",
+    "gpe",
+    "gu",
+    "guc",
+    "gur",
+    "guw",
+    "gv",
+    "ha",
+    "hak",
+    "haw",
+    "he",
+    "hi",
+    "hif",
+    "hr",
+    "hsb",
+    "ht",
+    "hu",
+    "hy",
+    "hyw",
+    "ia",
+    "id",
+    "ie",
+    "ig",
+    "ik",
+    "ilo",
+    "inh",
+    "io",
+    "is",
+    "it",
+    "iu",
+    "ja",
+    "jam",
+    "jbo",
+    "jv",
+    "ka",
+    "kaa",
+    "kab",
+    "kbd",
+    "kbp",
+    "kcg",
+    "kg",
+    "ki",
+    "kk",
+    "kl",
+    "km",
+    "kn",
+    "ko",
+    "koi",
+    "krc",
+    "ks",
+    "ksh",
+    "ku",
+    "kv",
+    "kw",
+    "ky",
+    "la",
+    "lad",
+    "lb",
+    "lbe",
+    "lez",
+    "lfn",
+    "lg",
+    "li",
+    "lij",
+    "lld",
+    "lmo",
+    "ln",
+    "lo",
+    "lt",
+    "ltg",
+    "lv",
+    "mad",
+    "mai",
+    "map-bms",
+    "mdf",
+    "mg",
+    "mhr",
+    "mi",
+    "min",
+    "mk",
+    "ml",
+    "mn",
+    "mni",
+    "mnw",
+    "mr",
+    "mrj",
+    "ms",
+    "mt",
+    "mwl",
+    "my",
+    "myv",
+    "mzn",
+    "nah",
+    "nap",
+    "nds",
+    "nds-nl",
+    "ne",
+    "new",
+    "nia",
+    "nl",
+    "nn",
+    "no",
+    "nov",
+    "nqo",
+    "nrm",
+    "nso",
+    "nv",
+    "ny",
+    "oc",
+    "olo",
+    "om",
+    "or",
+    "os",
+    "pa",
+    "pag",
+    "pam",
+    "pap",
+    "pcd",
+    "pcm",
+    "pdc",
+    "pfl",
+    "pi",
+    "pih",
+    "pl",
+    "pms",
+    "pnb",
+    "pnt",
+    "ps",
+    "pt",
+    "pwn",
+    "qu",
+    "rm",
+    "rmy",
+    "rn",
+    "ro",
+    "roa-rup",
+    "roa-tara",
+    "ru",
+    "rue",
+    "rw",
+    "sa",
+    "sah",
+    "sat",
+    "sc",
+    "scn",
+    "sco",
+    "sd",
+    "se",
+    "sg",
+    "sh",
+    "shi",
+    "shn",
+    "si",
+    "simple",
+    "sk",
+    "skr",
+    "sl",
+    "sm",
+    "smn",
+    "sn",
+    "so",
+    "sq",
+    "sr",
+    "srn",
+    "ss",
+    "st",
+    "stq",
+    "su",
+    "sv",
+    "sw",
+    "szl",
+    "szy",
+    "ta",
+    "tay",
+    "tcy",
+    "te",
+    "tet",
+    "tg",
+    "th",
+    "ti",
+    "tk",
+    "tl",
+    "tly",
+    "tn",
+    "to",
+    "tpi",
+    "tr",
+    "trv",
+    "ts",
+    "tt",
+    "tum",
+    "tw",
+    "ty",
+    "tyv",
+    "udm",
+    "ug",
+    "uk",
+    "ur",
+    "uz",
+    "ve",
+    "vec",
+    "vep",
+    "vi",
+    "vls",
+    "vo",
+    "wa",
+    "war",
+    "wo",
+    "wuu",
+    "xal",
+    "xh",
+    "xmf",
+    "yi",
+    "yo",
+    "za",
+    "zea",
+    "zgh",
+    "zh",
+    "zh-classical",
+    "zh-min-nan",
+    "zh-yue",
+    "zu",
+]
+
+# Source: for each Wikipedia language code (example shown for "ab"), aliases for namespaces -2 and 6 accessed via this API call:
+# https://ab.wikipedia.org/w/api.php?action=query&meta=siteinfo&siprop=namespacealiases|namespaces&format=json&formatversion=2
+# Retrieved: 2023-11-17
+MEDIA_ALIASES = {
+    "ab": ["Амедиа", "Афаил", "Изображение", "Медиа", "Файл"],
+    "ace": ["Alat", "Berkas", "Beureukaih", "Gambar"],
+    "ady": ["Медиа"],
+    "af": ["Beeld", "Lêer"],
+    "als": ["Bild", "Datei", "Medium"],
+    "alt": ["Изображение", "Медиа", "Файл"],
+    "am": ["ስዕል", "ፋይል"],
+    "ami": ["Faylo", "Mitiya", "图像", "图片", "圖像", "圖片", "媒体", "媒体文件", "媒体档案", "媒體", "媒體文件", "媒體檔案", "文件", "档案", "檔案"],
+    "an": ["Imachen", "Imagen"],
+    "ang": ["Biliþ", "Ymele"],
+    "anp": ["फाईल", "मीडिया"],
+    "ar": ["صورة", "ملف", "ميديا", "وسائط"],
+    "arc": ["ܠܦܦܐ", "ܡܝܕܝܐ"],
+    "ary": ["صورة", "فيشي", "ملف", "ميديا", "وسائط"],
+    "arz": ["صورة", "ملف", "ميديا", "وسائط"],
+    "as": ["चित्र", "চিত্র", "চিত্ৰ", "মাধ্যম"],
+    "ast": ["Archivu", "Ficheru", "Imagen", "Imaxe", "Imaxen", "Medios"],
+    "atj": ["Natisinahikaniwoc", "Tipatcimoctakewin"],
+    "av": ["Изображение", "Медиа", "Файл"],
+    "avk": ["Ewava", "Imagen", "Iyeltak", "Kanaca", "Mamind", "Изображение"],
+    "awa": ["फाइल", "मीडिया"],
+    "ay": ["Archivo", "Imagen", "Medio"],
+    "az": ["Fayl", "Mediya", "Şəkil"],
+    "azb": ["تصویر", "رسانه", "رسانه‌ای", "فایل", "مدیا"],
+    "ba": ["Изображение", "Медиа", "Рәсем", "Файл"],
+    "ban": ["Berkas", "Gambar", "Média"],
+    "bar": ["Bild", "Datei", "Medium"],
+    "bat-smg": ["Abruozdielis", "Medėjė", "Vaizdas"],
+    "bbc": ["Ugasan"],
+    "bcl": ["Ladawan", "Medio"],
+    "be": ["Выява", "Мультымедыя", "Файл"],
+    "be-tarask": ["Выява", "Мэдыя", "Файл"],
+    "bg": ["Картинка", "Медия", "Файл"],
+    "bh": ["चित्र", "मीडिया"],
+    "bjn": ["Barakas", "Berkas", "Gambar"],
+    "blk": ["ဖုဲင်", "မီဒီယာ"],
+    "bm": ["Fichier", "Média"],
+    "bn": ["চিত্র", "মিডিয়া"],
+    "bpy": ["ছবি", "মিডিয়া"],
+    "br": ["Restr", "Skeudenn"],
+    "bs": ["Datoteka", "Medija", "Mediji", "Slika"],
+    "bug": ["Berkas", "Gambar"],
+    "bxr": ["Изображение", "Меди", "Файл"],
+    "ca": ["Fitxer", "Imatge"],
+    "cbk-zam": ["Archivo", "Imagen", "Medio"],
+    "cdo": ["图像", "图片", "圖像", "圖片", "媒体", "媒体文件", "媒体档案", "媒體", "媒體文件", "媒體檔案", "文件", "档案", "檔案"],
+    "ce": ["Изображение", "Медиа", "Медйа", "Сурт", "Файл", "Хlум"],
+    "ceb": ["Imahen", "Medya", "Payl"],
+    "ch": ["Litratu"],
+    "ckb": ["میدیا", "پەڕگە"],
+    "co": ["Immagine"],
+    "crh": ["Fayl", "Resim", "Медиа", "Ресим", "Файл"],
+    "cs": ["Média", "Obrázok", "Soubor"],
+    "csb": ["Grafika", "Òbrôzk"],
+    "cu": ["Ви́дъ", "Видъ", "Дѣло", "Срѣдьства"],
+    "cv": ["Изображение", "Медиа", "Ӳкерчĕк"],
+    "cy": ["Delwedd"],
+    "da": ["Billede", "Fil"],
+    "dag": ["Lahabali kɔligu", "Miidiya"],
+    "de": ["Bild", "Datei", "Medium"],
+    "dga": ["Duoro bimbu zie", "Duoro kɔre"],
+    "din": ["Apamduööt", "Ciɛl"],
+    "diq": ["Dosya", "Medya"],
+    "dsb": ["Bild", "Dataja", "Medija", "Wobraz"],
+    "dty": ["चित्र", "मिडिया"],
+    "dv": ["މީޑިއާ", "ފައިލު", "ފައިލް"],
+    "el": ["Αρχείο", "Εικόνα", "Μέσο", "Μέσον"],
+    "eml": ["Immagine"],
+    "eo": ["Aŭdvidaĵo", "Dosiero"],
+    "es": ["Archivo", "Imagen", "Medio"],
+    "et": ["Fail", "Meedia", "Pilt"],
+    "eu": ["Fitxategi", "Irudi"],
+    "ext": ["Archivu", "Imagen", "Mediu"],
+    "fa": ["تصویر", "رسانه", "رسانه‌ای", "مدیا", "پرونده"],
+    "fat": ["Fael"],
+    "ff": ["Fichier", "Média"],
+    "fi": ["Kuva", "Tiedosto"],
+    "fiu-vro": ["Meediä", "Pilt"],
+    "fo": ["Miðil", "Mynd"],
+    "fon": ["Wékpo", "Yɛwliɖonuji"],
+    "fr": ["Fichier", "Média"],
+    "frp": ["Fichiér", "Mèdia", "Émâge"],
+    "frr": ["Bild", "Datei", "Medium"],
+    "fur": ["Figure", "Immagine"],
+    "fy": ["Ofbyld"],
+    "ga": ["Meán", "Íomhá"],
+    "gag": ["Dosya", "Dosye", "Mediya", "Medya", "Resim"],
+    "gan": ["图像", "图片", "圖像", "圖片", "媒体", "媒体文件", "媒体档案", "媒體", "媒體文件", "媒體檔案", "文件", "文檔", "档案", "��案"],
+    "gcr": ["Fiché", "Médja"],
+    "gd": ["Faidhle", "Meadhan"],
+    "gl": ["Arquivo", "Ficheiro", "Imagem", "Imaxe"],
+    "glk": ["تصویر", "رسانه", "رسانه‌ای", "فاىل", "مديا", "پرونده"],
+    "gn": ["Imagen", "Medio", "Ta'ãnga"],
+    "gom": ["फायल", "माध्यम", "मिडिया"],
+    "gor": ["Berkas", "Gambar"],
+    "got": ["𐍆𐌴𐌹𐌻𐌰"],
+    "gu": ["ચિત્ર", "દ્રશ્ય-શ્રાવ્ય (મિડિયા)"],
+    "guc": ["Anaajaalaa", "Ayaakuwapülee", "Imagen"],
+    "gur": ["Faali", "Miidiya"],
+    "guw": ["Wepo"],
+    "gv": ["Coadan", "Meanyn"],
+    "hak": ["图像", "图片", "圖像", "圖片", "媒体", "媒体文件", "媒体档案", "媒體", "媒體文件", "媒體檔案", "文件", "档案", "檔案"],
+    "haw": ["Kiʻi", "Pāpaho", "Waihona"],
+    "he": ["מדיה", "קו", "קובץ", "תמונה"],
+    "hi": ["चित्र", "मीडिया"],
+    "hif": ["file", "saadhan"],
+    "hr": ["DT", "Datoteka", "Mediji", "Slika"],
+    "hsb": ["Bild", "Dataja", "Wobraz"],
+    "ht": ["Fichye", "Imaj", "Medya"],
+    "hu": ["Fájl", "Kép", "Média"],
+    "hy": ["Մեդիա", "Պատկեր"],
+    "hyw": ["Մեդիա", "Պատկեր"],
+    "ia": ["Imagine", "Multimedia"],
+    "id": ["Berkas", "Gambar"],
+    "ig": ["Midia", "Nká", "Usòrò", "Ákwúkwó orünotu"],
+    "ilo": ["Midia", "Papeles"],
+    "inh": ["Изображение", "Медиа", "Файл"],
+    "io": ["Arkivo", "Imajo"],
+    "is": ["Miðill", "Mynd"],
+    "it": ["Immagine"],
+    "ja": ["ファイル", "メディア", "画像"],
+    "jbo": ["datnyvei", "velsku"],
+    "jv": ["Barkas", "Gambar", "Medhia", "Médhia"],
+    "ka": ["მედია", "სურათი", "ფაილი"],
+    "kaa": ["Fayl", "Su'wret", "Swret", "Taspa", "Сурет", "Таспа", "تاسپا", "سۋرەت"],
+    "kab": ["Tugna"],
+    "kbd": ["Медиа", "Файл"],
+    "kbp": ["Fichier", "Média"],
+    "kcg": ["Fail"],
+    "kg": ["Fisye"],
+    "kk": ["Swret", "Taspa", "Сурет", "Таспа", "تاسپا", "سۋرەت"],
+    "kl": ["Billede", "Fiileq", "Fil"],
+    "km": ["មីឌា", "មេឌា", "រូបភាព", "ឯកសារ"],
+    "kn": ["ಚಿತ್ರ", "ಮೀಡಿಯ"],
+    "ko": ["그림", "미디어", "파일"],
+    "koi": ["Изображение", "Медиа", "Файл"],
+    "krc": ["Изображение", "Медиа", "Файл"],
+    "ks": ["فَیِل", "میڈیا"],
+    "ksh": ["Beld", "Belld", "Bild", "Datei", "Medie", "Medium", "Meedije", "Meedijum"],
+    "ku": ["Medya", "Wêne", "میدیا", "پەڕگە"],
+    "kv": ["Изображение", "Медиа", "Файл"],
+    "kw": ["Restren"],
+    "ky": ["Медиа", "Файл"],
+    "la": ["Fasciculus", "Imago"],
+    "lad": ["Archivo", "Dossia", "Dosya", "Imagen", "Meddia", "Medya"],
+    "lb": ["Bild", "Fichier"],
+    "lbe": ["Изображение", "Медиа", "Сурат"],
+    "lez": ["Mediya", "Şəkil", "Изображение", "Медиа", "Файл"],
+    "lfn": ["Fix"],
+    "li": ["Aafbeilding", "Afbeelding", "Plaetje"],
+    "lij": ["Immaggine", "Immagine"],
+    "lld": ["Immagine"],
+    "lmo": ["Archivi", "Immagine", "Imàjine"],
+    "ln": ["Fichier", "Média"],
+    "lo": ["ສື່", "ສື່ອ", "ຮູບ"],
+    "lt": ["Medija", "Vaizdas"],
+    "ltg": ["Fails", "Medeja"],
+    "lv": ["Attēls"],
+    "mad": ["Bhengkek", "Gambar", "Mèḍia"],
+    "mai": ["फाइल", "मेडिया"],
+    "map-bms": ["Barkas", "Gambar", "Medhia", "Médhia"],
+    "mdf": ["Изображение", "Медиа", "Няйф"],
+    "mg": ["Média", "Rakitra", "Sary"],
+    "mhr": ["Изображение", "Медиа", "Файл"],
+    "min": ["Berkas", "Gambar"],
+    "mk": ["Медиум", "Медија", "Податотека", "Слика"],
+    "ml": ["ചി", "ചിത്രം", "പ്ര", "പ്രമാണം", "മീഡിയ"],
+    "mn": ["Зураг", "Медиа", "Файл"],
+    "mni": ["ꯃꯦꯗꯤꯌꯥ", "ꯐꯥꯏꯜ"],
+    "mnw": ["မဳဒဳယာ", "ဝှာင်"],
+    "mr": ["चित्र", "मिडिया"],
+    "mrj": ["Изображение", "Медиа", "Файл"],
+    "ms": ["Fail", "Imej"],
+    "mt": ["Medja", "Midja", "Stampa"],
+    "mwl": ["Arquivo", "Fexeiro", "Ficheiro", "Imagem", "Multimédia"],
+    "my": ["ဖိုင်", "မီဒီယာ"],
+    "myv": ["Артовкс", "Изображение", "Медия"],
+    "mzn": ["تصویر", "رسانه", "رسانه‌ای", "مدیا", "مه‌دیا", "پرونده"],
+    "nah": ["Imagen", "Mēdiatl", "Īxiptli"],
+    "nap": ["Fiùra", "Immagine"],
+    "nds": ["Bild", "Datei"],
+    "nds-nl": ["Afbeelding", "Bestaand", "Ofbeelding"],
+    "ne": ["चित्र", "मीडिया"],
+    "new": ["किपा", "माध्यम"],
+    "nia": ["Berkas", "Gambar"],
+    "nl": ["Afbeelding", "Bestand"],
+    "nn": ["Bilde", "Fil", "Filpeikar"],
+    "no": ["Bilde", "Fil", "Medium"],
+    "nqo": ["ߞߐߕߐ߮", "ߟߊߛߋߢߊߥߙߍ"],
+    "nrm": ["Fichier", "Média"],
+    "nso": ["Seswantšho"],
+    "nv": ["Eʼelyaaígíí"],
+    "oc": ["Fichièr", "Imatge", "Mèdia"],
+    "olo": ["Failu", "Kuva", "Medii"],
+    "or": ["ଫାଇଲ", "ମାଧ୍ୟମ"],
+    "os": ["Изображение", "Медиа", "Ныв", "Файл"],
+    "pa": ["ਤਸਵੀਰ", "ਮੀਡੀਆ"],
+    "pcd": ["Fichier", "Média"],
+    "pdc": ["Bild", "Datei", "Feil", "Medium"],
+    "pfl": ["Bild", "Dadai", "Datei", "Medium"],
+    "pi": ["पटिमा", "मीडिया"],
+    "pl": ["Grafika", "Plik"],
+    "pms": ["Figura", "Immagine"],
+    "pnb": ["تصویر", "فائل", "میڈیا"],
+    "pnt": ["Αρχείον", "Εικόνα", "Εικόναν", "Μέσον"],
+    "ps": ["انځور", "دوتنه", "رسنۍ"],
+    "pt": ["Arquivo", "Ficheiro", "Imagem", "Multimédia"],
+    "pwn": [
+        "mitiya",
+        "sineqetj a vecik",
+        "图像",
+        "图片",
+        "圖像",
+        "圖片",
+        "媒体",
+        "媒体文件",
+        "媒体档案",
+        "媒體",
+        "媒體文件",
+        "媒體檔案",
+        "文件",
+        "档案",
+        "檔案",
+    ],
+    "qu": ["Imagen", "Midya", "Rikcha"],
+    "rm": ["Bild", "Datoteca", "Multimedia"],
+    "rmy": ["Chitro", "Fişier", "Imagine", "Mediya"],
+    "rn": ["Dosiye"],
+    "ro": ["Fişier", "Fișier", "Imagine"],
+    "roa-rup": ["Fişier", "Fișier", "Imagine"],
+    "roa-tara": ["Immagine"],
+    "ru": ["Изображение", "Медиа", "Файл"],
+    "rue": ["Зображення", "Изображение", "Медиа", "Медіа", "Файл"],
+    "rw": ["Dosiye", "Itangazamakuru"],
+    "sa": ["चित्रं", "चित्रम्", "माध्यम", "माध्यमम्", "सञ्चिका"],
+    "sah": ["Билэ", "Изображение", "Миэдьийэ", "Ойуу"],
+    "sat": ["ᱢᱤᱰᱤᱭᱟ", "ᱨᱮᱫ"],
+    "sc": ["Immàgini"],
+    "scn": ["Immagine", "Mmàggini", "Mèdia"],
+    "sd": ["ذريعات", "عڪس", "فائل"],
+    "se": ["Bilde", "Fiila", "Kuva"],
+    "sg": ["Fichier", "Média"],
+    "sh": ["DT", "Datoteka", "Fotografija", "Medija", "Mediji", "Slika", "ДТ", "Датотека", "Медиј"],
+    "shi": ["Afaylu", "Midya"],
+    "shn": ["သိုဝ်ႇၶၢဝ်ႇ", "ၾၢႆႇ"],
+    "si": ["ගොනුව", "මාධ්‍යය", "රූපය"],
+    "sk": ["Médiá", "Obrázok", "Súbor"],
+    "skr": ["فائل", "میڈیا"],
+    "sl": ["Datoteka", "Slika"],
+    "smn": ["Kuva", "Tiätuvuárkká"],
+    "sq": ["Figura", "Skeda"],
+    "sr": [
+        "Datoteka",
+        "Fotografija",
+        "Medij",
+        "Medija",
+        "Slika",
+        "Датотека",
+        "Медиј",
+        "Медија",
+        "Слика",
+        "Фотографија",
+    ],
+    "srn": ["Afbeelding", "Gefre"],
+    "stq": ["Bielde", "Bild"],
+    "su": ["Gambar", "Média"],
+    "sv": ["Bild", "Fil"],
+    "sw": ["Faili", "Picha"],
+    "szl": ["Grafika", "Plik"],
+    "szy": ["myiti", "tangan", "图像", "图片", "圖像", "圖片", "媒体", "媒体文件", "媒体档案", "媒體", "媒體文件", "媒體檔案", "文件", "档案", "檔案"],
+    "ta": ["ஊடகம்", "படிமம்"],
+    "tay": [
+        "biru' na zayzyuwaw",
+        "biru’ na zayzyuwaw",
+        "media",
+        "图像",
+        "图片",
+        "圖像",
+        "圖片",
+        "媒体",
+        "媒体文件",
+        "媒体档案",
+        "媒體",
+        "媒體文件",
+        "媒體檔案",
+        "文件",
+        "档案",
+        "檔案",
+    ],
+    "tcy": ["ಫೈಲ್", "ಮಾದ್ಯಮೊ"],
+    "te": ["దస్త్రం", "ఫైలు", "బొమ్మ", "మీడియా"],
+    "tet": ["Arquivo", "Imagem", "Imajen"],
+    "tg": ["Акс", "Медиа"],
+    "th": ["ภาพ", "สื่อ", "ไฟล์"],
+    "ti": ["ሜድያ", "ፋይል"],
+    "tk": ["Faýl"],
+    "tl": ["Midya", "Talaksan"],
+    "tly": ["Fajl", "Medja"],
+    "tn": ["Pego", "Setshwantsho"],
+    "tpi": ["Fail"],
+    "tr": ["Dosya", "Medya", "Ortam", "Resim"],
+    "trv": [
+        "Meyti",
+        "Patas bntasan",
+        "图像",
+        "图片",
+        "圖像",
+        "圖片",
+        "媒体",
+        "媒体文件",
+        "媒体档案",
+        "媒體",
+        "媒體文件",
+        "媒體檔案",
+        "文件",
+        "档案",
+        "檔案",
+    ],
+    "tt": ["Räsem", "Изображение", "Медиа", "Рәсем", "Файл"],
+    "ty": ["Fichier", "Média"],
+    "tyv": ["Изображение", "Медиа", "Файл"],
+    "udm": ["Изображение", "Медиа", "Суред", "Файл"],
+    "ug": ["ھۆججەت", "ۋاسىتە"],
+    "uk": ["Зображення", "Изображение", "Медиа", "Медіа", "Файл"],
+    "ur": ["تصویر", "زریعہ", "فا��ل", "ملف", "میڈیا", "وسیط"],
+    "uz": ["Fayl", "Mediya", "Tasvir"],
+    "vec": ["Immagine", "Imàjine", "Mèdia"],
+    "vep": ["Fail", "Pilt"],
+    "vi": ["Hình", "Phương tiện", "Tập tin"],
+    "vls": ["Afbeelding", "Ofbeeldienge"],
+    "vo": ["Magod", "Nünamakanäd", "Ragiv"],
+    "wa": ["Imådje"],
+    "war": ["Fayl", "Medya", "Paypay"],
+    "wo": ["Dencukaay", "Xibaarukaay"],
+    "wuu": ["图像", "图片", "圖像", "圖片", "媒体", "媒体文件", "媒体档案", "媒體", "媒體文件", "媒體檔案", "文件", "档案", "檔案"],
+    "xal": ["Аһар", "Боомг", "Зург", "Изображение"],
+    "xmf": ["მედია", "სურათი", "ფაილი"],
+    "yi": ["בילד", "טעקע", "מעדיע", "תמונה"],
+    "yo": ["Amóhùnmáwòrán", "Fáìlì", "Àwòrán"],
+    "za": ["图像", "图片", "圖像", "圖片", "媒体", "媒体文件", "媒体档案", "媒體", "媒體文件", "媒體檔案", "文件", "档案", "檔案"],
+    "zea": ["Afbeelding", "Plaetje"],
+    "zgh": ["ⴰⴼⴰⵢⵍⵓ", "ⵎⵉⴷⵢⴰ"],
+    "zh": ["图像", "图片", "圖像", "圖片", "媒体", "媒体文件", "媒体档案", "媒體", "媒體文件", "媒體檔案", "文件", "档案", "檔案"],
+    "zh-classical": ["图像", "图片", "圖像", "圖片", "媒体", "媒体文件", "媒体档案", "媒體", "媒體文件", "媒體檔案", "文件", "档案", "檔案"],
+    "zh-min-nan": [
+        "Mûi-thé",
+        "tóng-àn",
+        "图像",
+        "图片",
+        "圖像",
+        "圖片",
+        "媒体",
+        "媒体文件",
+        "媒体档案",
+        "媒體",
+        "媒體文件",
+        "媒體檔案",
+        "文件",
+        "档案",
+        "檔案",
+    ],
+    "zh-yue": ["图", "图像", "圖", "圖像", "媒体", "媒體", "文件", "档", "档案", "檔", "檔案"],
+}
+
+# Source: for each Wikipedia language code (example shown for "ab"), aliases for namespace 14 accessed via this API call:
+# https://ab.wikipedia.org/w/api.php?action=query&meta=siteinfo&siprop=namespacealiases|namespaces&format=json&formatversion=2
+# Retrieved: 2023-11-17
+CAT_ALIASES = {
+    "ab": ["Акатегориа", "Категория"],
+    "ace": ["Kategori", "Kawan"],
+    "af": ["Kategorie"],
+    "als": ["Kategorie"],
+    "alt": ["Категория"],
+    "am": ["መደብ"],
+    "ami": ["Kasasiwasiw", "分类", "分類"],
+    "an": ["Categoría"],
+    "ang": ["Flocc"],
+    "anp": ["श्रेणी"],
+    "ar": ["تصنيف"],
+    "arc": ["ܣܕܪܐ"],
+    "ary": ["تصنيف"],
+    "arz": ["تصنيف"],
+    "as": ["CAT", "श्रेणी", "শ্রেণী", "শ্ৰেণী"],
+    "ast": ["Categoría"],
+    "atj": ["Tipanictawin"],
+    "av": ["Категория"],
+    "avk": ["Loma"],
+    "awa": ["श्रेणी"],
+    "ay": ["Categoría"],
+    "az": ["Kateqoriya"],
+    "azb": ["بؤلمه"],
+    "ba": ["Категория", "Төркөм"],
+    "ban": ["Kategori"],
+    "bar": ["Kategorie"],
+    "bat-smg": ["Kategorija", "Kateguorėjė"],
+    "bbc": ["Horong"],
+    "bcl": ["Kategorya"],
+    "be": ["Катэгорыя"],
+    "be-tarask": ["Катэгорыя"],
+    "bg": ["Категория"],
+    "bh": ["श्रेणी"],
+    "bjn": ["Kategori", "Tumbung"],
+    "blk": ["ကဏ္ဍ"],
+    "bm": ["Catégorie"],
+    "bn": ["বিষয়শ্রেণী"],
+    "bpy": ["থাক"],
+    "br": ["Rummad"],
+    "bs": ["Kategorija"],
+    "bug": ["Kategori"],
+    "bxr": ["Категори", "Категория"],
+    "ca": ["Categoria"],
+    "cbk-zam": ["Categoría"],
+    "cdo": ["分类", "分類"],
+    "ce": ["Кадегар", "Категори", "Тоба"],
+    "ceb": ["Kategoriya"],
+    "ch": ["Katigoria"],
+    "ckb": ["پ", "پۆل"],
+    "co": ["Categoria"],
+    "crh": ["Kategoriya", "Категория"],
+    "cs": ["Kategorie"],
+    "csb": ["Kategòrëjô"],
+    "cu": ["Категория", "Катигорїꙗ", "Катигорї"],
+    "cv": ["Категори"],
+    "cy": ["Categori"],
+    "da": ["Kategori"],
+    "dag": ["Pubu"],
+    "de": ["Kategorie"],
+    "dga": ["Gbuli"],
+    "din": ["Bekätakthook"],
+    "diq": ["Kategori", "Kategoriye"],
+    "dsb": ["Kategorija"],
+    "dty": ["श्रेणी"],
+    "dv": ["ޤިސްމު"],
+    "el": ["Κατηγορία"],
+    "eml": ["Categoria"],
+    "eo": ["Kategorio"],
+    "es": ["CAT", "Categoría"],
+    "et": ["Kategooria"],
+    "eu": ["Kategoria"],
+    "ext": ["Categoria", "Categoría"],
+    "fa": ["رده"],
+    "fat": ["Nkyekyεmu"],
+    "ff": ["Catégorie"],
+    "fi": ["Luokka"],
+    "fiu-vro": ["Katõgooria"],
+    "fo": ["Bólkur"],
+    "fon": ["Akpaxwé"],
+    "fr": ["Catégorie"],
+    "frp": ["Catègorie"],
+    "frr": ["Kategorie"],
+    "fur": ["Categorie"],
+    "fy": ["Kategory"],
+    "ga": ["Catagóir", "Rang"],
+    "gag": ["Kategori", "Kategoriya"],
+    "gan": ["分类", "分類"],
+    "gcr": ["Katégori"],
+    "gd": ["Roinn-seòrsa"],
+    "gl": ["Categoría"],
+    "glk": ["جرگه", "رده"],
+    "gn": ["Ñemohenda"],
+    "gom": ["वर्ग", "श्रेणी"],
+    "gor": ["Dalala"],
+    "got": ["𐌷𐌰𐌽𐍃𐌰"],
+    "gu": ["CAT", "શ્રે", "શ્રેણી"],
+    "guc": ["Akotchajülee sünülia"],
+    "gur": ["Buuri buuri"],
+    "guw": ["Adà"],
+    "gv": ["Ronney"],
+    "hak": ["分类", "分類"],
+    "haw": ["Māhele"],
+    "he": ["קט", "קטגוריה"],
+    "hi": ["श्र", "श्रेणी"],
+    "hif": ["vibhag"],
+    "hr": ["CT", "KT", "Kategorija"],
+    "hsb": ["Kategorija"],
+    "ht": ["Kategori"],
+    "hu": ["Kategória"],
+    "hy": ["Կատեգորիա"],
+    "hyw": ["Ստորոգութիւն"],
+    "ia": ["Categoria"],
+    "id": ["Kategori"],
+    "ie": ["Categorie"],
+    "ig": ["Ébéonọr", "Òtù"],
+    "ilo": ["Kategoria"],
+    "inh": ["ОагӀат"],
+    "io": ["Kategorio"],
+    "is": ["Flokkur"],
+    "it": ["CAT", "Categoria"],
+    "ja": ["カテゴリ"],
+    "jbo": ["klesi"],
+    "jv": ["Kategori"],
+    "ka": ["კატეგორია"],
+    "kaa": ["Kategoriya", "Sanat", "Санат", "سانات"],
+    "kab": ["Taggayt"],
+    "kbd": ["Категориэ", "Категория"],
+    "kbp": ["Catégorie"],
+    "kcg": ["Sa"],
+    "kg": ["Kalasi"],
+    "kk": ["Sanat", "Санат", "سانات"],
+    "kl": ["Kategori", "Sumut atassuseq"],
+    "km": ["ចំណាត់ក្រុម", "ចំណាត់ថ្នាក់ក្រុម", "ចំនាត់ថ្នាក់ក្រុម"],
+    "kn": ["ವರ್ಗ"],
+    "ko": ["분류"],
+    "koi": ["Категория"],
+    "krc": ["Категория"],
+    "ks": ["زٲژ"],
+    "ksh": ["Kategorie", "Katejori", "Kattejori", "Saachjrop", "Saachjropp", "Saachjrupp", "Sachjrop"],
+    "ku": ["Kategorî", "پۆل"],
+    "kv": ["Категория"],
+    "kw": ["Class", "Klass"],
+    "ky": ["Категория"],
+    "la": ["Categoria"],
+    "lad": ["Categoría", "Kateggoría", "Katēggoría"],
+    "lb": ["Kategorie"],
+    "lbe": ["Категория"],
+    "lez": ["Категория"],
+    "lfn": ["Categoria"],
+    "li": ["Categorie", "Kategorie"],
+    "lij": ["Categoria", "Categorîa"],
+    "lld": ["Categoria"],
+    "lmo": ["Categoria", "Categuria"],
+    "ln": ["Catégorie"],
+    "lo": ["ໝວດ"],
+    "lt": ["Kategorija"],
+    "ltg": ["Kategoreja"],
+    "lv": ["Kategorija"],
+    "mad": ["Bhângsa"],
+    "mai": ["CA", "श्रेणी"],
+    "map-bms": ["Kategori"],
+    "mdf": ["Категорие", "Категория"],
+    "mg": ["Catégorie", "Sokajy"],
+    "mhr": ["Категорий", "Категория"],
+    "min": ["Kategori"],
+    "mk": ["Категорија"],
+    "ml": ["വ", "വി", "വിഭാഗം", "വർഗ്ഗം"],
+    "mn": ["Ангилал"],
+    "mni": ["ꯃꯆꯥꯈꯥꯏꯕ"],
+    "mnw": ["ကဏ္ဍ"],
+    "mr": ["वर्ग"],
+    "mrj": ["Категори", "Категория"],
+    "ms": ["Kategori"],
+    "mt": ["Kategorija"],
+    "mwl": ["Catadorie", "Categoria"],
+    "my": ["ကဏ္ဍ"],
+    "myv": ["Категория"],
+    "mzn": ["رج", "رده"],
+    "nah": ["Categoría", "Neneuhcāyōtl"],
+    "nap": ["Categoria", "Categurìa"],
+    "nds": ["Kategorie"],
+    "nds-nl": ["Categorie", "Kategorie", "Kattegerie"],
+    "ne": ["श्रेणी"],
+    "new": ["पुचः"],
+    "nia": ["Kategori"],
+    "nl": ["Categorie"],
+    "nn": ["Kategori"],
+    "no": ["Kategori"],
+    "nqo": ["ߦߌߟߡߊ"],
+    "nrm": ["Catégorie"],
+    "nso": ["Setensele"],
+    "nv": ["Tʼááłáhági átʼéego"],
+    "oc": ["Categoria"],
+    "olo": ["Kategourii"],
+    "or": ["ବିଭାଗ", "ଶ୍ରେଣୀ"],
+    "os": ["Категори"],
+    "pa": ["ਸ਼੍ਰੇਣੀ"],
+    "pcd": ["Catégorie"],
+    "pcm": ["Katigori"],
+    "pdc": ["Abdeeling", "Kategorie"],
+    "pfl": ["Kadegorie", "Kategorie", "Sachgrubb"],
+    "pi": ["विभाग"],
+    "pl": ["Kategoria"],
+    "pms": ["Categorìa"],
+    "pnb": ["گٹھ"],
+    "pnt": ["Κατηγορίαν"],
+    "ps": ["وېشنيزه"],
+    "pt": ["Categoria"],
+    "pwn": ["pinapapilipiliqan", "分类", "分類"],
+    "qu": ["Katiguriya"],
+    "rm": ["Categoria"],
+    "rmy": ["Shopni"],
+    "rn": ["Umuce"],
+    "ro": ["Categorie"],
+    "roa-rup": ["Categorie"],
+    "roa-tara": ["Categoria"],
+    "ru": ["К", "Категория"],
+    "rue": ["Категория", "Катеґорія"],
+    "rw": ["Ikiciro"],
+    "sa": ["वर्गः"],
+    "sah": ["Категория"],
+    "sat": ["ᱛᱷᱚᱠ"],
+    "sc": ["Categoria"],
+    "scn": ["Catigurìa"],
+    "sd": ["زمرو"],
+    "se": ["Kategoriija"],
+    "sg": ["Catégorie"],
+    "sh": ["KAT", "KT", "Kategorija", "КАТ", "КТ", "Категорија"],
+    "shi": ["Taggayt"],
+    "shn": ["ပိူင်ထၢၼ်ႈ"],
+    "si": ["ප්‍රවර්ගය"],
+    "sk": ["Kategória"],
+    "skr": ["ونکی"],
+    "sl": ["Kategorija"],
+    "smn": ["Luokka"],
+    "sq": ["Kategori", "Kategoria"],
+    "sr": ["Kategorija", "Категорија"],
+    "srn": ["Categorie", "Guru"],
+    "stq": ["Kategorie"],
+    "su": ["Kategori"],
+    "sv": ["Kategori"],
+    "sw": ["Jamii"],
+    "szl": ["Kategoria", "Kategoryjo"],
+    "szy": ["kakuniza", "分类", "分類"],
+    "ta": ["பகுப்பு"],
+    "tay": ["zyuwaw na", "分类", "分類"],
+    "tcy": ["ವರ್ಗೊ"],
+    "te": ["వర్గం"],
+    "tet": ["Kategoria", "Kategoría"],
+    "tg": ["Гурӯҳ"],
+    "th": ["หมวดหมู่"],
+    "ti": ["መደብ"],
+    "tk": ["Kategoriýa"],
+    "tl": ["Kategorya", "Kaurian"],
+    "tly": ["Tispir"],
+    "tn": ["Karolo"],
+    "tpi": ["Grup"],
+    "tr": ["KAT", "Kategori"],
+    "trv": ["Snakun", "分类", "分類"],
+    "tt": ["Törkem", "Категория", "Төркем"],
+    "tw": ["Nkyekyεmu"],
+    "ty": ["Catégorie"],
+    "tyv": ["Аңгылал", "Категория"],
+    "udm": ["Категория"],
+    "ug": ["تۈر"],
+    "uk": ["Категория", "Категорія"],
+    "ur": ["زمرہ"],
+    "uz": ["Kategoriya", "Turkum"],
+    "vec": ["Categoria"],
+    "vep": ["Kategorii"],
+    "vi": ["Thể loại"],
+    "vls": ["Categorie"],
+    "vo": ["Klad"],
+    "wa": ["Categoreye"],
+    "war": ["Kaarangay"],
+    "wo": ["Catégorie", "Wàll"],
+    "wuu": ["分类", "分類"],
+    "xal": ["Янз", "Әәшл"],
+    "xmf": ["კატეგორია"],
+    "yi": ["קאַטעגאָריע", "קאטעגאריע"],
+    "yo": ["Ẹ̀ka"],
+    "za": ["分类", "分類"],
+    "zea": ["Categorie"],
+    "zgh": ["ⴰⵙⵎⵉⵍ"],
+    "zh": ["CAT", "分类", "分類"],
+    "zh-classical": ["CAT", "分类", "分類"],
+    "zh-min-nan": ["Lūi-pia̍t", "分类", "分類"],
+    "zh-yue": ["分类", "分類", "类", "類"],
+}
+
+# Note that Wikimedia servers have rate limited downloaders and they are capping the number of per-ip connections to 2
+# Their mirror sites do not have this cap: https://dumps.wikimedia.org/mirrors.html
+_HOST = "https://dumps.wikimedia.org"
+_URL_PATH_SEGMENT = "/{lang}wiki/{date}/"
+_INFO_FILE = "dumpstatus.json"
+
+
+_VERSION = datasets.Version("4.0.0", "")
+
+
+class WikipediaConfig(datasets.BuilderConfig):
+    """BuilderConfig for Wikipedia."""
+
+    def __init__(self, language=None, date=None, host=_HOST, version=_VERSION, **kwargs):
+        """BuilderConfig for Wikipedia.
+        Args:
+            language (str): Language code for the Wikipedia dump to use.
+            date (str): Date of the Wikipedia dump in YYYYMMDD format. A list of
+                available dates can be found at https://dumps.wikimedia.org/enwiki/.
+            host (str, defaults to 'https://dumps.wikimedia.org'): URL of the server that hosts the Wikipedia dump.
+                It defaults to the official Wikimedia host, but this has rate limited downloaders and is capping the
+                number of per-IP connections to 2.
+                To parallelize data download, use a mirror: https://dumps.wikimedia.org/mirrors.html
+            **kwargs: Keyword arguments forwarded to super.
+        """
+        super().__init__(
+            name=f"{date}.{language}",
+            description=f"Wikipedia dataset for {language}, parsed from {date} dump.",
+            version=version,
+            **kwargs,
+        )
+        self.date = date
+        self.language = language
+        self.host = host.rstrip("/")
+
+
+class Wikipedia(datasets.GeneratorBasedBuilder):
+    """Wikipedia dataset."""
+
+    # Use mirror (your.org) to avoid download caps.
+    BUILDER_CONFIG_CLASS = WikipediaConfig
+
+    def _info(self):
+        return datasets.DatasetInfo(
+            description=_DESCRIPTION,
+            features=datasets.Features(
+                {
+                    "id": datasets.Value("string"),
+                    "url": datasets.Value("string"),
+                    "title": datasets.Value("string"),
+                    "text": datasets.Value("string"),
+                }
+            ),
+            homepage=_HOMEPAGE,
+            citation=_CITATION,
+        )
+
+    def _get_base_url(self):
+        return self.config.host + _URL_PATH_SEGMENT.format(
+            lang=self.config.language.replace("-", "_"), date=self.config.date
+        )
+
+    def _split_generators(self, dl_manager):
+        # Download dump status info file
+        info_url = self._get_base_url() + _INFO_FILE
+        info_path = dl_manager.download_and_extract(info_url)
+        with open(info_path, encoding="utf-8") as f:
+            dump_info = json.load(f)
+        multistream_dump_info = dump_info["jobs"]["articlesmultistreamdump"]
+        if multistream_dump_info["status"] != "done":
+            raise FileNotFoundError(
+                f"Specified dump ({self._get_base_url()}) multistream status is not 'done':"
+                f" {multistream_dump_info['status']}"
+            )
+
+        # Download index and multistream XML data files
+        total_bytes = 0
+        index_urls = []
+        xml_urls = []
+        for fname, info in multistream_dump_info["files"].items():
+            if ".txt" in fname:
+                index_urls.append(self._get_base_url() + fname)
+            elif ".xml" in fname:
+                total_bytes += info["size"]
+                xml_urls.append(self._get_base_url() + fname)
+        data_urls = [*zip(sorted(index_urls), sorted(xml_urls))]  # Parallelize data downloading
+        data_paths = dl_manager.download(data_urls)
+
+        # Parallelize over concatenated multiple compressed streams (with 100 pages each)
+        filepaths, starts, ends = [], [], []
+        for index_path, xml_path in data_paths:
+            index = _extract_index(index_path)
+            for start, end in _pairwise(index):
+                filepaths.append(xml_path)
+                starts.append(start)
+                ends.append(end)
+
+        return [
+            datasets.SplitGenerator(
+                name=datasets.Split.TRAIN,
+                gen_kwargs={
+                    "filepaths": filepaths,
+                    "starts": starts,
+                    "ends": ends,
+                },
+            )
+        ]
+
+    def _generate_examples(self, filepaths, starts, ends):
+        # No shuffle anymore
+        for filepath, start, end in zip(filepaths, starts, ends):
+            for id_, title, raw_content in _extract_content(filepath, start, end):
+                yield from _clean_content((id_, title, raw_content), self.config.language)
+
+
+def _extract_index(filepath):
+    with open(filepath, "rb") as compressed_file:
+        binary_file = bz2.BZ2File(filename=compressed_file)
+        text_file = io.TextIOWrapper(binary_file, encoding="utf-8", newline="")
+        return sorted({int(row[0]) for row in csv.reader(text_file, delimiter=":")})
+
+
+def _pairwise(index):
+    starts, ends = itertools.tee(index)
+    _ = next(ends, None)
+    return itertools.zip_longest(starts, ends)
+
+
+def _extract_content(filepath, start, end):
+    """Extract articles from a single stream of a multistream WikiMedia XML file."""
+    logger.info("generating examples from = %s", filepath)
+    with open(filepath, "rb") as compressed_file:
+        compressed_file.seek(start)
+        compressed_data = compressed_file.read(end - start if end else -1)
+        binary_data = bz2.BZ2Decompressor().decompress(compressed_data)
+        # Enclose within a single root node to avoid ParseError: junk after document element
+        binary_data = b"<mediawiki>" + binary_data + b"</mediawiki>"
+        with io.StringIO(binary_data.decode(encoding="utf-8")) as text_stream:
+            for _, elem in etree.iterparse(text_stream):
+                if not elem.tag.endswith("page"):
+                    continue
+                namespace = elem.tag[:-4]
+                ns = elem.find(f"./{namespace}ns").text
+                redirect = elem.find(f"./{namespace}redirect")
+                # Filter pages that are not in the "main" namespace or that are redirects
+                if ns != "0" or redirect is not None:
+                    elem.clear()
+                    continue
+                id_ = elem.find(f"./{namespace}id").text
+                title = elem.find(f"./{namespace}title").text
+                raw_content = elem.find(f"./{namespace}revision/{namespace}text").text
+                elem.clear()
+                # Filter empty pages
+                if raw_content is None:
+                    continue
+                yield id_, title, raw_content
+
+
+def _clean_content(inputs, language):
+    """Clean raw wikicode to extract text."""
+    id_, title, raw_content = inputs
+    try:
+        text = _parse_and_clean_wikicode(raw_content, parser=mwparserfromhell, language=language)
+    except mwparserfromhell.parser.ParserError as e:
+        logger.error("mwparserfromhell ParseError: %s", e)
+        return
+    if not text:
+        return
+    url = _construct_url(title, language)
+    yield id_, {"id": id_, "url": url, "title": title, "text": text}
+
+
+def _parse_and_clean_wikicode(raw_content, parser, language):
+    """Strip formatting and unwanted sections from raw page content."""
+    wikicode = parser.parse(raw_content)
+
+    # Filters for magic words that are parser instructions -- e.g., __NOTOC__
+    re_rm_magic = re.compile("__[A-Z]*__", flags=re.UNICODE)
+
+    # Filters for file/image links.
+    media_prefixes = "|".join(["File", "Image", "Media"] + MEDIA_ALIASES.get(language, []))
+    re_rm_wikilink = re.compile(f"^(?:{media_prefixes}):", flags=re.IGNORECASE | re.UNICODE)
+
+    def rm_wikilink(obj):
+        return bool(re_rm_wikilink.match(str(obj.title)))
+
+    # Filters for references and tables
+    def rm_tag(obj):
+        return str(obj.tag) in {"ref", "table"}
+
+    # Leave category links in-place but remove the category prefixes
+    cat_prefixes = "|".join(["Category"] + CAT_ALIASES.get(language, []))
+    re_clean_wikilink = re.compile(f"^(?:{cat_prefixes}):", flags=re.IGNORECASE | re.UNICODE)
+
+    def is_category(obj):
+        return bool(re_clean_wikilink.match(str(obj.title)))
+
+    def clean_wikilink(obj):
+        text = obj.__strip__()
+        text = re.sub(re_clean_wikilink, "", text)
+        obj.text = text
+
+    def try_replace_obj(obj):
+        try:
+            clean_wikilink(obj)
+        except ValueError:
+            # For unknown reasons, objects are sometimes not found.
+            pass
+
+    def try_remove_obj(obj, section):
+        try:
+            section.remove(obj)
+        except ValueError:
+            # For unknown reasons, objects are sometimes not found.
+            pass
+
+    section_text = []
+    # Filter individual sections to clean.
+    for section in wikicode.get_sections(flat=True, include_lead=True, include_headings=True):
+        for obj in section.ifilter_wikilinks(recursive=True):
+            if rm_wikilink(obj):
+                try_remove_obj(obj, section)
+            elif is_category(obj):
+                try_replace_obj(obj)
+        for obj in section.ifilter_tags(matches=rm_tag, recursive=True):
+            try_remove_obj(obj, section)
+
+        section_text.append(re.sub(re_rm_magic, "", section.strip_code().strip()))
+    return "\n\n".join(section_text)
+
+
+def _construct_url(title, language):
+    # See: https://meta.wikimedia.org/wiki/Help:URL
+    return f"https://{language}.wikipedia.org/wiki/{quote(title)}"

requirements.txt

@@ -0,0 +1,4 @@
+datatrove[io]==0.6.0
+gradio==5.49.1
+mwparserfromhell==0.7.2
+datasets==4.0.0