feat: add ifeval parser

llmdataparser/ifeval_parser.py

@@ -0,0 +1,94 @@
+from dataclasses import dataclass
+from typing import Any, ClassVar, List
+
+from llmdataparser.base_parser import HuggingFaceDatasetParser, HuggingFaceParseEntry
+from llmdataparser.prompts import IFEVAL_SYSTEM_PROMPT  # You'll need to create this
+
+
+@dataclass(frozen=True, kw_only=True, slots=True)
+class IFEvalParseEntry(HuggingFaceParseEntry):
+    """Custom entry class for IFEval, with fields specific to this dataset parser."""
+
+    key: int
+    instruction_id_list: List[str]
+    kwargs: dict[str, Any]
+
+    @classmethod
+    def create(
+        cls,
+        prompt: str,
+        answer: str,
+        raw_question: str,
+        raw_answer: str,
+        key: int,
+        instruction_id_list: List[str],
+        kwargs: dict[str, Any],
+        task_name: str,
+    ) -> "IFEvalParseEntry":
+        return cls(
+            prompt=prompt,
+            answer=answer,
+            raw_question=raw_question,
+            raw_answer=raw_answer,
+            key=key,
+            instruction_id_list=instruction_id_list,
+            kwargs=kwargs,
+            task_name=task_name,
+        )
+
+
+class IFEvalDatasetParser(HuggingFaceDatasetParser[IFEvalParseEntry]):
+    """Parser for the IFEval dataset."""
+
+    _data_source: ClassVar[str] = "google/IFEval"
+    _default_task: ClassVar[str] = "default"
+    _task_names: ClassVar[list[str]] = ["default"]
+    _default_system_prompt: ClassVar[str] = IFEVAL_SYSTEM_PROMPT
+
+    def process_entry(
+        self, row: dict[str, Any], task_name: str | None = None, **kwargs: Any
+    ) -> IFEvalParseEntry:
+        """Process a single IFEval entry."""
+        # Extract fields from the row
+        key = row["key"]
+        raw_question = row["prompt"]  # The prompt is the raw question in this case
+        instruction_id_list = row["instruction_id_list"]
+        kwargs_data = row["kwargs"]
+
+        # For IFEval, we don't have explicit answers in the dataset
+        # We'll use empty strings as placeholders
+        answer = ""
+        raw_answer = ""
+
+        # Combine system prompt with the instruction prompt
+        prompt = f"{self._system_prompt}\n\n{raw_question}"
+
+        # Use task_name if provided, otherwise use default
+        task = task_name or self._get_current_task(row)
+
+        return IFEvalParseEntry.create(
+            prompt=prompt,
+            answer=answer,
+            raw_question=raw_question,
+            raw_answer=raw_answer,
+            key=key,
+            instruction_id_list=instruction_id_list,
+            kwargs=kwargs_data,
+            task_name=task,
+        )
+
+
+if __name__ == "__main__":
+    # Example usage
+    parser = IFEvalDatasetParser()
+    parser.load()
+    parser.parse()
+
+    parsed_data = parser.get_parsed_data
+    if parsed_data:
+        example = parsed_data[0]
+        print("\nExample parsed entry:")
+        print(f"Key: {example.key}")
+        print(f"Prompt: {example.prompt}")
+        print(f"Instruction IDs: {example.instruction_id_list}")
+        print(f"kwargs: {example.kwargs}")

tests/test_ifeval_parser.py

@@ -0,0 +1,91 @@
+import pytest
+
+from llmdataparser.ifeval_parser import IFEvalDatasetParser, IFEvalParseEntry
+
+
+@pytest.fixture
+def sample_ifeval_entries():
+    """Create sample IFEval dataset entries for testing."""
+    return [
+        {
+            "key": 1,
+            "prompt": "Write a function to calculate factorial.",
+            "instruction_id_list": ["math_001", "programming_001"],
+            "kwargs": {"difficulty": "medium", "category": "mathematics"},
+        },
+        {
+            "key": 2,
+            "prompt": "Explain quantum computing.",
+            "instruction_id_list": ["physics_001"],
+            "kwargs": {"difficulty": "hard", "category": "physics"},
+        },
+    ]
+
+
+@pytest.fixture
+def ifeval_parser():
+    """Create an IFEval parser instance."""
+    return IFEvalDatasetParser()
+
+
+def test_ifeval_parse_entry_creation_valid():
+    """Test valid creation of IFEvalParseEntry."""
+    entry = IFEvalParseEntry.create(
+        prompt="Test system prompt\n\nTest instruction",
+        answer="",  # IFEval doesn't have answers
+        raw_question="Test instruction",
+        raw_answer="",
+        key=1,
+        instruction_id_list=["test_001", "test_002"],
+        kwargs={"difficulty": "easy"},
+        task_name="default",
+    )
+
+    assert isinstance(entry, IFEvalParseEntry)
+    assert entry.prompt == "Test system prompt\n\nTest instruction"
+    assert entry.answer == ""
+    assert entry.key == 1
+    assert entry.instruction_id_list == ["test_001", "test_002"]
+    assert entry.kwargs == {"difficulty": "easy"}
+    assert entry.task_name == "default"
+
+
+def test_process_entry_ifeval(ifeval_parser, sample_ifeval_entries):
+    """Test processing entries in IFEval parser."""
+    entry = ifeval_parser.process_entry(sample_ifeval_entries[0])
+
+    assert isinstance(entry, IFEvalParseEntry)
+    assert entry.key == 1
+    assert entry.instruction_id_list == ["math_001", "programming_001"]
+    assert entry.kwargs == {"difficulty": "medium", "category": "mathematics"}
+    assert entry.raw_question == "Write a function to calculate factorial."
+    assert entry.answer == ""  # IFEval doesn't have answers
+    assert entry.task_name == "default"
+
+
+def test_parser_initialization(ifeval_parser):
+    """Test initialization of IFEval parser."""
+    assert ifeval_parser._data_source == "google/IFEval"
+    assert ifeval_parser._default_task == "default"
+    assert ifeval_parser.task_names == ["default"]
+    assert (
+        ifeval_parser.get_huggingface_link
+        == "https://huggingface.co/datasets/google/IFEval"
+    )
+
+
+@pytest.mark.integration
+def test_load_dataset(ifeval_parser):
+    """Test loading the IFEval dataset."""
+    ifeval_parser.load(split="train")
+    assert ifeval_parser.raw_data is not None
+    assert ifeval_parser.split_names == ["train"]
+    assert ifeval_parser._current_task == "default"
+
+
+def test_parser_string_representation(ifeval_parser):
+    """Test string representation of IFEval parser."""
+    repr_str = str(ifeval_parser)
+    assert "IFEvalDatasetParser" in repr_str
+    assert "google/IFEval" in repr_str
+    assert "not loaded" in repr_str