refactor: tmli/tw_legal parser

llmdataparser/tmlu_parser.py

@@ -169,27 +169,6 @@ class TMLUDatasetParser(HuggingFaceDatasetParser[TMLUParseEntry]):
                 implementation="custom_subject_accuracy",
                 primary=True,
             ),
-            EvaluationMetric.create(
-                name="per_difficulty_accuracy",
-                type="classification",
-                description="Accuracy broken down by test difficulty levels",
-                implementation="custom_difficulty_accuracy",
-                primary=False,
-            ),
-            EvaluationMetric.create(
-                name="confusion_matrix",
-                type="classification",
-                description="Distribution of predicted vs actual answers",
-                implementation="datasets.load_metric('confusion_matrix')",
-                primary=False,
-            ),
-            EvaluationMetric.create(
-                name="explanation_quality",
-                type="text",
-                description="Quality assessment of model explanations when available",
-                implementation="custom_explanation_metric",
-                primary=False,
-            ),
         ]
 
 

llmdataparser/tw_legal_parser.py

@@ -1,7 +1,12 @@
 from dataclasses import dataclass
 from typing import Any, Final
 
-from llmdataparser.base_parser import HuggingFaceDatasetParser, HuggingFaceParseEntry
+from llmdataparser.base_parser import (
+    DatasetDescription,
+    EvaluationMetric,
+    HuggingFaceDatasetParser,
+    HuggingFaceParseEntry,
+)
 from llmdataparser.prompts import TW_LEGAL_SYSTEM_PROMPT
 
 TW_LEGAL_VALID_ANSWERS: Final[set[str]] = {"A", "B", "C", "D"}
@@ -70,6 +75,35 @@ class TWLegalDatasetParser(HuggingFaceDatasetParser[TWLegalParseEntry]):
             task_name=task,
         )
 
+    def get_dataset_description(self) -> DatasetDescription:
+        """Returns description of the Taiwan Legal Benchmark dataset."""
+        return DatasetDescription.create(
+            name="Taiwan Legal Benchmark",
+            language="Traditional Chinese",
+            purpose="Evaluate models on Taiwan-specific legal knowledge and understanding",
+            source="Taiwan Bar Examination questions",
+            format="Multiple choice questions (A/B/C/D)",
+            characteristics=(
+                "Contains questions from Taiwan's bar examination, testing understanding "
+                "of Taiwan's legal system, terminology, and concepts"
+            ),
+            citation="""
+                url={https://huggingface.co/datasets/lianghsun/tw-legal-benchmark-v1}
+            }""",
+        )
+
+    def get_evaluation_metrics(self) -> list[EvaluationMetric]:
+        """Returns recommended evaluation metrics for Taiwan Legal Benchmark."""
+        return [
+            EvaluationMetric.create(
+                name="accuracy",
+                type="classification",
+                description="Overall percentage of correctly answered legal questions",
+                implementation="datasets.load_metric('accuracy')",
+                primary=True,
+            ),
+        ]
+
 
 if __name__ == "__main__":
     # Example usage

tests/test_tmlu_parser.py

@@ -187,23 +187,10 @@ def test_get_evaluation_metrics(tmlu_parser):
     """Test evaluation metrics generation."""
     metrics = tmlu_parser.get_evaluation_metrics()
 
-    assert len(metrics) == 5  # Check total number of metrics
+    assert len(metrics) == 2  # Check total number of metrics
 
     # Check primary metrics
     primary_metrics = [m for m in metrics if m.primary]
     assert len(primary_metrics) == 2
     assert any(m.name == "accuracy" for m in primary_metrics)
     assert any(m.name == "per_subject_accuracy" for m in primary_metrics)
-
-    # Check specific metric properties
-    accuracy_metric = next(m for m in metrics if m.name == "accuracy")
-    assert accuracy_metric.type == "classification"
-    assert "datasets.load_metric('accuracy')" in accuracy_metric.implementation
-
-    # Check non-primary metrics
-    non_primary_metrics = {m.name for m in metrics if not m.primary}
-    assert non_primary_metrics == {
-        "per_difficulty_accuracy",
-        "confusion_matrix",
-        "explanation_quality",
-    }

tests/test_tw_legal_parser.py

@@ -138,3 +138,27 @@ def test_system_prompt_override(tw_legal_parser):
 
     entry = parser.process_entry(test_entry)
     assert custom_prompt in entry.prompt
+
+
+def test_get_dataset_description(tw_legal_parser):
+    """Test getting dataset description for Taiwan Legal parser."""
+    description = tw_legal_parser.get_dataset_description()
+
+    assert description.name == "Taiwan Legal Benchmark"
+    assert description.language == "Traditional Chinese"
+    assert "Taiwan's legal system" in description.characteristics
+    assert (
+        "huggingface.co/datasets/lianghsun/tw-legal-benchmark-v1"
+        in description.citation
+    )
+
+
+def test_get_evaluation_metrics(tw_legal_parser):
+    """Test getting evaluation metrics for Taiwan Legal parser."""
+    metrics = tw_legal_parser.get_evaluation_metrics()
+
+    assert len(metrics) == 1
+    metric = metrics[0]
+    assert metric.name == "accuracy"
+    assert metric.type == "classification"
+    assert metric.primary is True