tests/test_hub_features.py

# Copyright 2023-present the HuggingFace Inc. team.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import copy

import pytest
import torch
from huggingface_hub import ModelCard
from transformers import AutoModelForCausalLM

from peft import AutoPeftModelForCausalLM, BoneConfig, LoraConfig, PeftConfig, PeftModel, TaskType, get_peft_model

from .testing_utils import hub_online_once


PEFT_MODELS_TO_TEST = [("peft-internal-testing/test-lora-subfolder", "test")]


class PeftHubFeaturesTester:
    # TODO remove when/if Hub is more stable
    @pytest.mark.xfail(reason="Test is flaky on CI", raises=ValueError)
    def test_subfolder(self):
        r"""
        Test if subfolder argument works as expected
        """
        for model_id, subfolder in PEFT_MODELS_TO_TEST:
            config = PeftConfig.from_pretrained(model_id, subfolder=subfolder)

            model = AutoModelForCausalLM.from_pretrained(
                config.base_model_name_or_path,
            )
            model = PeftModel.from_pretrained(model, model_id, subfolder=subfolder)

            assert isinstance(model, PeftModel)


class TestLocalModel:
    def test_local_model_saving_no_warning(self, recwarn, tmp_path):
        # When the model is saved, the library checks for vocab changes by
        # examining `config.json` in the model path.
        # However, previously, those checks only covered huggingface hub models.
        # This test makes sure that the local `config.json` is checked as well.
        # If `save_pretrained` could not find the file, it will issue a warning.
        model_id = "facebook/opt-125m"
        model = AutoModelForCausalLM.from_pretrained(model_id)
        local_dir = tmp_path / model_id
        model.save_pretrained(local_dir)
        del model

        base_model = AutoModelForCausalLM.from_pretrained(local_dir)
        peft_config = LoraConfig()
        peft_model = get_peft_model(base_model, peft_config)
        peft_model.save_pretrained(local_dir)

        for warning in recwarn.list:
            assert "Could not find a config file" not in warning.message.args[0]


class TestBaseModelRevision:
    def test_save_and_load_base_model_revision(self, tmp_path):
        r"""
        Test saving a PeftModel with a base model revision and loading with AutoPeftModel to recover the same base
        model
        """
        lora_config = LoraConfig(r=8, lora_alpha=16, lora_dropout=0.0)
        test_inputs = torch.arange(10).reshape(-1, 1)

        base_model_id = "peft-internal-testing/tiny-random-BertModel"
        revision = "v2.0.0"

        base_model_revision = AutoModelForCausalLM.from_pretrained(base_model_id, revision=revision).eval()
        peft_model_revision = get_peft_model(base_model_revision, lora_config, revision=revision)
        output_revision = peft_model_revision(test_inputs).logits

        # sanity check: the model without revision should be different
        base_model_no_revision = AutoModelForCausalLM.from_pretrained(base_model_id, revision="main").eval()
        # we need a copy of the config because otherwise, we are changing in-place the `revision` of the previous config and model
        lora_config_no_revision = copy.deepcopy(lora_config)
        lora_config_no_revision.revision = "main"
        peft_model_no_revision = get_peft_model(base_model_no_revision, lora_config_no_revision, revision="main")
        output_no_revision = peft_model_no_revision(test_inputs).logits
        assert not torch.allclose(output_no_revision, output_revision)

        # check that if we save and load the model, the output corresponds to the one with revision
        peft_model_revision.save_pretrained(tmp_path / "peft_model_revision")
        peft_model_revision_loaded = AutoPeftModelForCausalLM.from_pretrained(tmp_path / "peft_model_revision").eval()

        assert peft_model_revision_loaded.peft_config["default"].revision == revision

        output_revision_loaded = peft_model_revision_loaded(test_inputs).logits
        assert torch.allclose(output_revision, output_revision_loaded)

    # TODO remove when/if Hub is more stable
    @pytest.mark.xfail(reason="Test is flaky on CI", raises=ValueError)
    def test_load_different_peft_and_base_model_revision(self, tmp_path):
        r"""
        Test loading an AutoPeftModel from the hub where the base model revision and peft revision differ
        """
        base_model_id = "hf-internal-testing/tiny-random-BertModel"
        base_model_revision = None
        peft_model_id = "peft-internal-testing/tiny-random-BertModel-lora"
        peft_model_revision = "v1.2.3"

        peft_model = AutoPeftModelForCausalLM.from_pretrained(peft_model_id, revision=peft_model_revision).eval()

        assert peft_model.peft_config["default"].base_model_name_or_path == base_model_id
        assert peft_model.peft_config["default"].revision == base_model_revision


class TestModelCard:
    @pytest.mark.parametrize(
        "model_id, peft_config, tags, excluded_tags, pipeline_tag",
        [
            (
                "hf-internal-testing/tiny-random-Gemma3ForCausalLM",
                LoraConfig(),
                ["transformers", "base_model:adapter:hf-internal-testing/tiny-random-Gemma3ForCausalLM", "lora"],
                [],
                None,
            ),
            (
                "hf-internal-testing/tiny-random-Gemma3ForCausalLM",
                BoneConfig(),
                ["transformers", "base_model:adapter:hf-internal-testing/tiny-random-Gemma3ForCausalLM"],
                ["lora"],
                None,
            ),
            (
                "hf-internal-testing/tiny-random-BartForConditionalGeneration",
                LoraConfig(),
                [
                    "transformers",
                    "base_model:adapter:hf-internal-testing/tiny-random-BartForConditionalGeneration",
                    "lora",
                ],
                [],
                None,
            ),
            (
                "hf-internal-testing/tiny-random-Gemma3ForCausalLM",
                LoraConfig(task_type=TaskType.CAUSAL_LM),
                ["transformers", "base_model:adapter:hf-internal-testing/tiny-random-Gemma3ForCausalLM", "lora"],
                [],
                "text-generation",
            ),
        ],
    )
    @pytest.mark.parametrize(
        "pre_tags",
        [
            ["tag1", "tag2"],
            [],
        ],
    )
    def test_model_card_has_expected_tags(
        self, model_id, peft_config, tags, excluded_tags, pipeline_tag, pre_tags, tmp_path
    ):
        """Make sure that PEFT sets the tags in the model card automatically and correctly.
        This is important so that a) the models are searchable on the Hub and also 2) some features depend on it to
        decide how to deal with them (e.g., inference).

        Makes sure that the base model tags are still present (if there are any).
        """
        with hub_online_once(model_id):
            base_model = AutoModelForCausalLM.from_pretrained(model_id)

            if pre_tags:
                base_model.add_model_tags(pre_tags)

            peft_model = get_peft_model(base_model, peft_config)
            save_path = tmp_path / "adapter"

            peft_model.save_pretrained(save_path)

            model_card = ModelCard.load(save_path / "README.md")
            assert set(tags).issubset(set(model_card.data.tags))

            if excluded_tags:
                assert set(excluded_tags).isdisjoint(set(model_card.data.tags))

            if pre_tags:
                assert set(pre_tags).issubset(set(model_card.data.tags))

            if pipeline_tag:
                assert model_card.data.pipeline_tag == pipeline_tag

    @pytest.fixture
    def custom_model_cls(self):
        class MyNet(torch.nn.Module):
            def __init__(self):
                super().__init__()
                self.l1 = torch.nn.Linear(10, 20)
                self.l2 = torch.nn.Linear(20, 1)

            def forward(self, X):
                return self.l2(self.l1(X))

        return MyNet

    def test_custom_models_dont_have_transformers_tag(self, custom_model_cls, tmp_path):
        base_model = custom_model_cls()
        peft_config = LoraConfig(target_modules="all-linear")
        peft_model = get_peft_model(base_model, peft_config)

        peft_model.save_pretrained(tmp_path)

        model_card = ModelCard.load(tmp_path / "README.md")

        assert model_card.data.tags is not None
        assert "transformers" not in model_card.data.tags

    def test_custom_peft_type_does_not_raise(self, tmp_path):
        # Passing a string value as peft_type value in the config is valid, so it should work.
        # See https://github.com/huggingface/peft/issues/2634
        model_id = "hf-internal-testing/tiny-random-Gemma3ForCausalLM"
        with hub_online_once(model_id):
            base_model = AutoModelForCausalLM.from_pretrained(model_id)
            peft_config = LoraConfig()

            # We simulate a custom PEFT type by using a string value of an existing method.  This skips the need for
            # registering a new method but tests the case where we pass a string value instead of an enum.
            peft_type = "LORA"
            peft_config.peft_type = peft_type

            peft_model = get_peft_model(base_model, peft_config)
            peft_model.save_pretrained(tmp_path)