refactor: add category to parser

app.py

@@ -252,8 +252,24 @@ def update_metric_details(metric_name: str, parser_name: str) -> str:
 
 def create_interface() -> gr.Blocks:
     """Create and return the Gradio interface."""
-    with gr.Blocks() as demo:
-        gr.Markdown("# LLM Evaluation Dataset Parser")
+    with gr.Blocks(css="footer {display: none !important}") as demo:
+        # Add header section with purpose and GitHub info
+        gr.Markdown("""
+            # LLM Evaluation Dataset Parser
+
+            ### 🎯 Purpose
+            A unified interface for parsing and exploring various LLM benchmark datasets (MMLU, MMLU-Pro, GSM8k, and more).
+            This tool helps researchers and developers to:
+            - Easily explore different benchmark datasets
+            - Access standardized parsing for multiple dataset formats
+            - View dataset descriptions and evaluation metrics
+
+            ### 🔗 Links
+            - [GitHub Repository](https://github.com/jeff52415/LLMDataParser)
+            - [Documentation](https://github.com/jeff52415/LLMDataParser#readme)
+
+            ---
+        """)
 
         # State management
         parser_state = gr.State("")

llmdataparser/base_parser.py

@@ -9,6 +9,18 @@ import datasets
 T = TypeVar("T", bound="ParseEntry")
 
 
+# Add this after the DatasetCategory definition
+VALID_CATEGORIES = {
+    "Math",
+    "General Knowledge and Reasoning",
+    "Programming",
+    "MultiLingual",
+    "Taiwan",
+    "Advanced Reasoning",
+    "Legal",
+}
+
+
 @dataclass(frozen=True, kw_only=True, slots=True)
 class ParseEntry:
     """A simple base class for entries, customizable by each dataset parser."""
@@ -28,6 +40,7 @@ class DatasetDescription:
     source: str
     language: str
     format: str
+    category: list[str]
     characteristics: str
     citation: str | None = None
     additional_info: dict[str, Any] | None = None
@@ -40,16 +53,23 @@ class DatasetDescription:
         source: str,
         language: str,
         format: str,
+        category: list[str],
         characteristics: str,
         citation: str | None = None,
         additional_info: dict[str, Any] | None = None,
     ) -> "DatasetDescription":
+        # Validate that all categories are valid DatasetCategory values
+        for item in category:
+            assert (
+                item in VALID_CATEGORIES
+            ), f"Category '{item}' is not a valid category. Valid categories are: {VALID_CATEGORIES}"
         return cls(
             name=name,
             purpose=purpose,
             source=source,
             language=language,
             format=format,
+            category=category,
             characteristics=characteristics,
             citation=citation,
             additional_info=additional_info,

llmdataparser/bbh_parser.py

@@ -106,6 +106,7 @@ class BBHDatasetParser(HuggingFaceDatasetParser[BBHParseEntry]):
                 "significantly improved through chain-of-thought prompting. The dataset "
                 "includes 23 core tasks plus additional related tasks."
             ),
+            category=["Advanced Reasoning"],
             citation=(
                 "@article{suzgun2022challenging,\n"
                 "  title={Challenging BIG-Bench Tasks and Whether Chain-of-Thought Can Solve Them},\n"

llmdataparser/gsm8k_parser.py

@@ -89,6 +89,7 @@ class GSM8KDatasetParser(HuggingFaceDatasetParser[GSM8KParseEntry]):
             source="OpenAI",
             language="English",
             format="Word problems with step-by-step solutions and numerical answers",
+            category=["Math"],
             characteristics=(
                 "Collection of 8.5K grade school math word problems that require "
                 "multi-step reasoning. Problems gradually increase in difficulty "

llmdataparser/humaneval_parser.py

@@ -88,6 +88,7 @@ class HumanEvalDatasetParser(HuggingFaceDatasetParser[HumanEvalParseEntry]):
             source="OpenAI",
             language="Python",
             format="Function signatures with docstrings and unit tests",
+            category=["Programming"],
             characteristics=(
                 "Collection of 164 hand-written Python programming problems. Each problem "
                 "includes a function signature, docstring, example test cases, and hidden unit "
@@ -186,6 +187,7 @@ class HumanEvalDatasetPlusParser(HumanEvalDatasetParser):
             source="EvalPlus",
             language="Python",
             format="Function signatures with docstrings and comprehensive test suites",
+            category=["Programming"],
             characteristics=(
                 "Significantly enhanced version of HumanEval with 80x more test cases. "
                 "Includes extensive edge cases, boundary conditions, stress tests, and "

llmdataparser/ifeval_parser.py

@@ -90,6 +90,7 @@ class IFEvalDatasetParser(HuggingFaceDatasetParser[IFEvalParseEntry]):
             source="Google Research",
             language="English (BCP-47 en)",
             format="Verifiable instruction prompts with automated evaluation criteria",
+            category=["Programming"],
             characteristics=(
                 "Collection of approximately 500 verifiable instructions designed to evaluate "
                 "language models' instruction-following capabilities. Instructions include "

llmdataparser/math_parser.py

@@ -97,6 +97,7 @@ class MATHDatasetParser(HuggingFaceDatasetParser[MATHParseEntry]):
             source="Hendrycks et al., UC Berkeley (NeurIPS 2021)",
             language="English",
             format="Competition mathematics problems with step-by-step solutions",
+            category=["Math"],
             characteristics=(
                 "Collection of 12,500 challenging competition mathematics problems designed to "
                 "evaluate mathematical reasoning. Problems include step-by-step solutions that "

llmdataparser/mbpp_parser.py

@@ -95,6 +95,7 @@ class MBPPDatasetParser(HuggingFaceDatasetParser[MBPPParseEntry]):
             purpose="A benchmark for evaluating code generation capabilities using entry-level Python programming problems",
             source="https://github.com/google-research/google-research/tree/master/mbpp",
             language="English and Python",
+            category=["Programming"],
             format="Task descriptions in English with corresponding Python solutions and automated test cases",
             characteristics=(
                 "Contains approximately 1,000 crowd-sourced Python programming problems "

llmdataparser/mgsm_parser.py

@@ -106,6 +106,7 @@ class MGSMDatasetParser(HuggingFaceDatasetParser[MGSMParseEntry]):
             source="https://huggingface.co/datasets/juletxara/mgsm",
             language="Multilingual (11 languages)",
             format="Word problems with numerical answers and solution steps",
+            category=["Math", "MultiLingual"],
             characteristics=(
                 "Human-translated version of 250 GSM8K problems into 10 additional languages. "
                 "Each problem includes the original question from GSM8K, its translations, "

llmdataparser/mmlu_parser.py

@@ -212,6 +212,7 @@ class BaseMMLUDatasetParser(MMLUDatasetParser):
             purpose="Evaluate models' extensive world knowledge and problem-solving abilities across diverse branches of knowledge",
             source="https://huggingface.co/datasets/cais/mmlu",
             language="English",
+            category=["General Knowledge and Reasoning"],
             format="Multiple choice questions with four options (A, B, C, D)",
             characteristics=(
                 "Comprehensive evaluation benchmark spanning humanities, social sciences, hard sciences, "
@@ -332,6 +333,7 @@ class MMLUReduxDatasetParser(MMLUDatasetParser):
             source="https://huggingface.co/datasets/edinburgh-dawg/mmlu-redux",
             language="English",
             format="Multiple choice questions with four options (A, B, C, D)",
+            category=["General Knowledge and Reasoning"],
             characteristics=(
                 "A carefully curated subset of 3,000 questions across 30 MMLU subjects, "
                 "manually re-annotated to identify and classify various types of errors. "
@@ -494,6 +496,7 @@ class TMMLUPlusDatasetParser(MMLUDatasetParser):
             purpose="Evaluate language models' understanding and reasoning capabilities in Traditional Chinese across diverse subjects",
             source="https://huggingface.co/datasets/ikala/tmmluplus",
             language="Traditional Chinese",
+            category=["General Knowledge and Reasoning", "Taiwan"],
             format="Multiple choice questions with four options (A, B, C, D)",
             characteristics=(
                 "A comprehensive evaluation benchmark featuring 66 subjects from elementary "
@@ -621,6 +624,7 @@ class MMLUProDatasetParser(HuggingFaceDatasetParser[MMLUProParseEntry]):
             purpose="Provide a more robust and challenging multi-task language understanding benchmark with enhanced reasoning requirements",
             source="https://huggingface.co/datasets/TIGER-Lab/MMLU-Pro",
             language="English",
+            category=["General Knowledge and Reasoning", "Advanced Reasoning"],
             format="Multiple choice questions with up to 10 options (expanded from original 4)",
             characteristics=(
                 "A more challenging version of MMLU containing 12K complex questions across various "

llmdataparser/tmlu_parser.py

@@ -130,6 +130,7 @@ class TMLUDatasetParser(HuggingFaceDatasetParser[TMLUParseEntry]):
             language="Traditional Chinese",
             purpose="Evaluate models on Taiwan-specific educational and professional knowledge",
             source="Various Taiwan standardized tests and professional certifications",
+            category=["Taiwan", "General Knowledge and Reasoning"],
             format="Multiple choice questions (A/B/C/D)",
             characteristics=(
                 "Covers various subjects including Advanced Subjects Test (AST), "

llmdataparser/tw_legal_parser.py

@@ -82,6 +82,7 @@ class TWLegalDatasetParser(HuggingFaceDatasetParser[TWLegalParseEntry]):
             language="Traditional Chinese",
             purpose="Evaluate models on Taiwan-specific legal knowledge and understanding",
             source="Taiwan Bar Examination questions",
+            category=["Taiwan", "General Knowledge and Reasoning", "Legal"],
             format="Multiple choice questions (A/B/C/D)",
             characteristics=(
                 "Contains questions from Taiwan's bar examination, testing understanding "