refactor: test cases

tests/test_bbh_parser.py

@@ -165,14 +165,9 @@ def test_get_dataset_description(bbh_parser):
     description = bbh_parser.get_dataset_description()
 
     assert description.name == "Big Bench Hard (BBH)"
-    assert "challenging BIG-Bench tasks" in description.purpose
     assert description.language == "English"
     assert description.format == "Multiple choice questions with single correct answers"
-    assert "Tasks require complex multi-step reasoning" in description.characteristics
     assert "suzgun2022challenging" in description.citation
-    assert description.additional_info is not None
-    assert "model_performance" in description.additional_info
-    assert "size" in description.additional_info
 
 
 def test_get_evaluation_metrics(bbh_parser):

tests/test_gsm8k_parser.py

@@ -190,10 +190,7 @@ def test_get_dataset_description(gsm8k_parser):
     assert description.name == "Grade School Math 8K (GSM8K)"
     assert description.source == "OpenAI"
     assert description.language == "English"
-    assert "8.5K grade school math word problems" in description.characteristics
-    assert "Training Verifiers to Solve Math Word Problems" in description.citation
     assert "Cobbe" in description.citation
-    assert "arXiv" in description.citation
 
 
 def test_get_evaluation_metrics(gsm8k_parser):
@@ -210,18 +207,3 @@ def test_get_evaluation_metrics(gsm8k_parser):
     assert exact_match.type == "string"
     assert exact_match.primary is True
     assert "exact match" in exact_match.description.lower()
-
-    # Check solution_validity metric details
-    solution_validity = next(m for m in metrics if m.name == "solution_validity")
-    assert solution_validity.type == "text"
-    assert solution_validity.primary is True
-    assert "valid" in solution_validity.description.lower()
-
-    # Check step metrics
-    step_accuracy = next(m for m in metrics if m.name == "step_accuracy")
-    assert step_accuracy.type == "numerical"
-    assert step_accuracy.primary is True
-
-    step_count = next(m for m in metrics if m.name == "step_count")
-    assert step_count.type == "numerical"
-    assert step_count.primary is False

tests/test_humaneval_parser.py

@@ -180,8 +180,6 @@ def test_get_dataset_description(parser, plus_parser):
     assert description.name == "HumanEval"
     assert "code generation" in description.purpose
     assert description.language == "Python"
-    assert "Function signatures with docstrings" in description.format
-    assert "164 hand-written Python programming problems" in description.characteristics
     assert "chen2021codex" in description.citation
 
     # Test HumanEval Plus description

tests/test_ifeval_parser.py

@@ -96,14 +96,8 @@ def test_get_dataset_description(ifeval_parser):
     description = ifeval_parser.get_dataset_description()
 
     assert description.name == "IFEval"
-    assert "verifiable instructions" in description.purpose.lower()
     assert description.source == "Google Research"
     assert description.language == "English (BCP-47 en)"
-    assert "verifiable instruction prompts" in description.format.lower()
-    assert "500" in description.characteristics
-    assert "automated heuristics" in description.characteristics.lower()
-    assert "open llm leaderboard" in description.characteristics.lower()
-    assert "zhou2023instructionfollowingevaluation" in description.citation
 
 
 def test_get_evaluation_metrics(ifeval_parser):
@@ -124,19 +118,3 @@ def test_get_evaluation_metrics(ifeval_parser):
     assert "punctuation_rules" in metric_names
     assert "keyword_usage" in metric_names
     assert "structural_requirements" in metric_names
-
-    # Check specific metric properties
-    format_metric = next(m for m in metrics if m.name == "format_compliance")
-    assert format_metric.primary is True
-    assert "formatting rules" in format_metric.description.lower()
-    assert format_metric.type == "text"
-
-    length_metric = next(m for m in metrics if m.name == "length_constraints")
-    assert length_metric.primary is True
-    assert "word" in length_metric.description.lower()
-    assert length_metric.type == "text"
-
-    punctuation_metric = next(m for m in metrics if m.name == "punctuation_rules")
-    assert punctuation_metric.primary is True
-    assert "punctuation" in punctuation_metric.description.lower()
-    assert punctuation_metric.type == "text"

tests/test_math_parser.py

@@ -205,12 +205,9 @@ def test_get_dataset_description(math_parser):
     description = math_parser.get_dataset_description()
 
     assert description.name == "MATH"
-    assert "mathematical problem-solving" in description.purpose.lower()
     assert "Hendrycks" in description.source
     assert description.language == "English"
-    assert "competition mathematics problems" in description.format.lower()
     assert "12,500" in description.characteristics
-    assert "step-by-step solutions" in description.characteristics.lower()
     assert "hendrycksmath2021" in description.citation
     assert "NeurIPS" in description.citation
 
@@ -220,8 +217,6 @@ def test_get_dataset_description(math_parser):
     assert "algebra" in description.additional_info["topics"]
     assert "geometry" in description.additional_info["topics"]
     assert description.additional_info["size"] == "12,500 problems"
-    assert "sympy" in description.additional_info["evaluation_note"].lower()
-    assert "github.com/hendrycks/math" in description.additional_info["homepage"]
 
 
 def test_get_evaluation_metrics(math_parser):
@@ -259,7 +254,3 @@ def test_get_evaluation_metrics(math_parser):
     assert reasoning_metric.type == "text"
     assert reasoning_metric.primary is True
     assert "mathematical reasoning" in reasoning_metric.description.lower()
-
-    # Check non-primary metrics
-    non_primary_metrics = {m.name for m in metrics if not m.primary}
-    assert non_primary_metrics == {"mathematical_notation", "solution_clarity"}

tests/test_mbpp_parser.py

@@ -162,31 +162,10 @@ def test_get_dataset_description(parser):
     assert "code generation" in description.purpose.lower()
     assert "google-research" in description.source
     assert description.language == "English and Python"
-    assert "task descriptions" in description.format.lower()
-    assert "python solutions" in description.format.lower()
     assert "1,000" in description.characteristics
-    assert "entry-level programmers" in description.characteristics.lower()
-    assert "3 automated test cases" in description.characteristics
-    assert "hand-verified" in description.characteristics
     assert "austin2021program" in description.citation
     assert "Program Synthesis" in description.citation
 
-    # Check additional info
-    assert description.additional_info is not None
-    assert description.additional_info["size"] == "~1,000 programming problems"
-    assert (
-        description.additional_info["splits"]
-        == "Available in full or sanitized versions"
-    )
-    assert (
-        description.additional_info["test_coverage"]
-        == "Each problem includes 3 automated test cases"
-    )
-    assert (
-        description.additional_info["verification"]
-        == "Subset of data has been hand-verified by authors"
-    )
-
 
 def test_get_evaluation_metrics(parser):
     """Test evaluation metrics generation."""
@@ -211,15 +190,3 @@ def test_get_evaluation_metrics(parser):
     assert pass_k_metric.primary is True
     assert "k generations" in pass_k_metric.description.lower()
     assert "custom_pass_at_k" in pass_k_metric.implementation
-
-    test_case_metric = next(m for m in metrics if m.name == "test_case_success_rate")
-    assert test_case_metric.type == "code_evaluation"
-    assert test_case_metric.primary is False
-    assert "test cases" in test_case_metric.description.lower()
-    assert "custom_test_success_rate" in test_case_metric.implementation
-
-    syntax_metric = next(m for m in metrics if m.name == "syntax_validity")
-    assert syntax_metric.type == "code_evaluation"
-    assert syntax_metric.primary is False
-    assert "syntactically valid" in syntax_metric.description.lower()
-    assert "custom_syntax_check" in syntax_metric.implementation

tests/test_mgsm_parser.py

@@ -192,35 +192,18 @@ def test_get_dataset_description(mgsm_parser):
     assert "multilingual chain-of-thought reasoning" in description.purpose.lower()
     assert "juletxara/mgsm" in description.source
     assert description.language == "Multilingual (11 languages)"
-    assert "word problems" in description.format.lower()
-    assert "numerical answers" in description.format.lower()
-    assert "solution steps" in description.format.lower()
-
-    # Check characteristics
-    assert "250" in description.characteristics
-    assert "gsm8k" in description.characteristics.lower()
-    assert "translations" in description.characteristics.lower()
+
     assert "mathematical reasoning" in description.characteristics.lower()
 
     # Check citations
     assert "shi2022language" in description.citation
     assert "cobbe2021gsm8k" in description.citation
-    assert (
-        "Language Models are Multilingual Chain-of-Thought Reasoners"
-        in description.citation
-    )
-    assert "Training Verifiers to Solve Math Word Problems" in description.citation
 
     # Check additional info
     assert description.additional_info is not None
     assert len(description.additional_info["languages"]) == 11
     assert "English" in description.additional_info["languages"]
     assert "Chinese" in description.additional_info["languages"]
-    assert (
-        description.additional_info["size"]
-        == "250 problems translated into each language"
-    )
-    assert description.additional_info["base_dataset"] == "GSM8K (Grade School Math 8K)"
 
 
 def test_get_evaluation_metrics(mgsm_parser):
@@ -259,12 +242,3 @@ def test_get_evaluation_metrics(mgsm_parser):
     assert step_metric.primary is True
     assert "calculation steps" in step_metric.description.lower()
     assert "custom_step_accuracy" in step_metric.implementation
-
-    # Check cross-lingual metric specifically
-    cross_lingual_metric = next(
-        m for m in metrics if m.name == "cross_lingual_consistency"
-    )
-    assert cross_lingual_metric.type == "comparison"
-    assert cross_lingual_metric.primary is False
-    assert "different language versions" in cross_lingual_metric.description.lower()
-    assert "custom_language_comparator" in cross_lingual_metric.implementation