refactor: bbh/tmlu test case

tests/test_bbh_parser.py

@@ -160,74 +160,38 @@ def test_different_tasks_parsing(bbh_parser, task_name):
     assert all(isinstance(entry.answer, str) for entry in parsed_data)
 
 
-def test_get_evaluation_metrics(bbh_parser):
-    """Test evaluation metrics structure and content."""
-    metrics = bbh_parser.get_evaluation_metrics()
-
-    # Check basic structure
-    assert isinstance(metrics, list)
-    assert len(metrics) > 0
-
-    # Check each metric has required fields
-    required_fields = ["name", "type", "description", "implementation", "primary"]
-    for metric in metrics:
-        for field in required_fields:
-            assert field in metric, f"Missing field {field} in metric {metric['name']}"
-
-        # Check field types
-        assert isinstance(metric["name"], str)
-        assert isinstance(metric["type"], str)
-        assert isinstance(metric["description"], str)
-        assert isinstance(metric["implementation"], str)
-        assert isinstance(metric["primary"], bool)
-
-    # Check specific metrics exist
-    metric_names = {m["name"] for m in metrics}
-    expected_metrics = {
-        "accuracy",
-        "human_eval_delta",
-        "per_task_accuracy",
-        "exact_match",
-    }
-    assert expected_metrics.issubset(metric_names)
-
-    # Check primary metrics
-    primary_metrics = {m["name"] for m in metrics if m["primary"]}
-    assert "accuracy" in primary_metrics
-    assert "human_eval_delta" in primary_metrics
-
-
-def test_dataset_description_citation_format(bbh_parser):
-    """Test that the citation in dataset description is properly formatted."""
+def test_get_dataset_description(bbh_parser):
+    """Test dataset description generation."""
     description = bbh_parser.get_dataset_description()
-    citation = description["citation"]
-
-    # Check citation structure
-    assert citation.startswith("@article{")
-    assert "title=" in citation
-    assert "author=" in citation
-    assert "journal=" in citation
-    assert "year=" in citation
 
-    # Check specific author formatting
-    assert "Suzgun, Mirac" in citation
-    assert "Wei, Jason" in citation
-    assert "and Wei, Jason" in citation  # Should be last author
-    assert "and and" not in citation  # No double "and"
+    assert description.name == "Big Bench Hard (BBH)"
+    assert "challenging BIG-Bench tasks" in description.purpose
+    assert description.language == "English"
+    assert description.format == "Multiple choice questions with single correct answers"
+    assert "Tasks require complex multi-step reasoning" in description.characteristics
+    assert "suzgun2022challenging" in description.citation
+    assert description.additional_info is not None
+    assert "model_performance" in description.additional_info
+    assert "size" in description.additional_info
 
 
-def test_evaluation_metrics_implementations(bbh_parser):
-    """Test that evaluation metric implementations are properly specified."""
+def test_get_evaluation_metrics(bbh_parser):
+    """Test evaluation metrics generation."""
     metrics = bbh_parser.get_evaluation_metrics()
 
-    for metric in metrics:
-        impl = metric["implementation"]
-
-        if "evaluate.load" in impl:
-            # Check standard metric format
-            assert impl.startswith("evaluate.load('")
-            assert impl.endswith("')")
-        elif "custom_" in impl:
-            # Check custom metric format
-            assert impl.startswith("custom_")
-            assert len(impl) > 7  # More than just "custom_"
+    assert len(metrics) == 4  # Check total number of metrics
+
+    # Check primary metrics
+    primary_metrics = [m for m in metrics if m.primary]
+    assert len(primary_metrics) == 2
+    assert any(m.name == "accuracy" for m in primary_metrics)
+    assert any(m.name == "human_eval_delta" for m in primary_metrics)
+
+    # Check specific metric properties
+    accuracy_metric = next(m for m in metrics if m.name == "accuracy")
+    assert accuracy_metric.type == "classification"
+    assert "evaluate.load('accuracy')" in accuracy_metric.implementation
+
+    # Check non-primary metrics
+    assert any(m.name == "per_task_accuracy" and not m.primary for m in metrics)
+    assert any(m.name == "exact_match" and not m.primary for m in metrics)

tests/test_tmlu_parser.py

@@ -170,76 +170,40 @@ def test_metadata_handling(tmlu_parser, sample_tmlu_entries):
     assert entry.metadata["source"] == "AST chinese - 108"
 
 
-def test_dataset_description(tmlu_parser):
-    """Test dataset description contains all required fields."""
+def test_get_dataset_description(tmlu_parser):
+    """Test dataset description generation."""
     description = tmlu_parser.get_dataset_description()
 
-    required_fields = [
-        "name",
-        "version",
-        "language",
-        "purpose",
-        "source",
-        "format",
-        "size",
-        "domain",
-        "characteristics",
-        "reference",
-    ]
-
-    for field in required_fields:
-        assert field in description, f"Missing required field: {field}"
+    assert description.name == "Taiwan Multiple-choice Language Understanding (TMLU)"
+    assert description.language == "Traditional Chinese"
+    assert "Taiwan-specific educational" in description.purpose
+    assert "Various Taiwan standardized tests" in description.source
+    assert description.format == "Multiple choice questions (A/B/C/D)"
+    assert "Advanced Subjects Test (AST)" in description.characteristics
+    assert "DBLP:journals/corr/abs-2403-20180" in description.citation
 
-    assert description["language"] == "Traditional Chinese"
-    assert "TMLU" in description["name"]
-    assert "miulab/tmlu" in description["reference"]
-    assert "AST" in description["characteristics"]
-    assert "GSAT" in description["characteristics"]
 
-
-def test_evaluation_metrics(tmlu_parser):
-    """Test evaluation metrics structure and content."""
+def test_get_evaluation_metrics(tmlu_parser):
+    """Test evaluation metrics generation."""
     metrics = tmlu_parser.get_evaluation_metrics()
 
-    # Check if we have metrics defined
-    assert len(metrics) > 0
+    assert len(metrics) == 5  # Check total number of metrics
 
-    # Check structure of each metric
-    required_metric_fields = [
-        "name",
-        "type",
-        "description",
-        "implementation",
-        "primary",
-    ]
+    # Check primary metrics
+    primary_metrics = [m for m in metrics if m.primary]
+    assert len(primary_metrics) == 2
+    assert any(m.name == "accuracy" for m in primary_metrics)
+    assert any(m.name == "per_subject_accuracy" for m in primary_metrics)
 
-    for metric in metrics:
-        for field in required_metric_fields:
-            assert field in metric, f"Missing required field in metric: {field}"
-
-        # Type checks
-        assert isinstance(metric["name"], str)
-        assert isinstance(metric["type"], str)
-        assert isinstance(metric["description"], str)
-        assert isinstance(metric["implementation"], str)
-        assert isinstance(metric["primary"], bool)
-
-    # Check for TMLU-specific metrics
-    metric_names = {m["name"] for m in metrics}
-    expected_metrics = {
-        "accuracy",
-        "per_subject_accuracy",
+    # Check specific metric properties
+    accuracy_metric = next(m for m in metrics if m.name == "accuracy")
+    assert accuracy_metric.type == "classification"
+    assert "datasets.load_metric('accuracy')" in accuracy_metric.implementation
+
+    # Check non-primary metrics
+    non_primary_metrics = {m.name for m in metrics if not m.primary}
+    assert non_primary_metrics == {
         "per_difficulty_accuracy",
+        "confusion_matrix",
         "explanation_quality",
     }
-
-    for expected in expected_metrics:
-        assert expected in metric_names, f"Missing expected metric: {expected}"
-
-    # Verify primary metrics
-    primary_metrics = [m for m in metrics if m["primary"]]
-    assert (
-        len(primary_metrics) >= 2
-    )  # Should have at least accuracy and per_subject_accuracy
-    assert any(m["name"] == "accuracy" for m in primary_metrics)
-    assert any(m["name"] == "per_subject_accuracy" for m in primary_metrics)