File size: 3,673 Bytes
302920f |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 |
# Copyright 2023-present the HuggingFace Inc. team.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import logging
import platform
import re
import pytest
def pytest_addoption(parser):
parser.addoption("--regression", action="store_true", default=False, help="run regression tests")
def pytest_configure(config):
config.addinivalue_line("markers", "regression: mark regression tests")
# Errors from transformers deprecations
logger = logging.getLogger("transformers")
class ErrorOnDeprecation(logging.Handler):
def emit(self, record):
msg = record.getMessage().lower()
if "deprecat" in msg or "future" in msg:
if "torch_dtype" not in msg:
# let's ignore the torch_dtype => dtype deprecation for now
raise AssertionError(f"**Transformers Deprecation**: {msg}")
# Add our handler
handler = ErrorOnDeprecation()
logger.addHandler(handler)
logger.setLevel(logging.WARNING)
def pytest_collection_modifyitems(config, items):
if config.getoption("--regression"):
return
skip_regression = pytest.mark.skip(reason="need --regression option to run regression tests")
for item in items:
if "regression" in item.keywords:
item.add_marker(skip_regression)
# TODO: remove this once support for PyTorch 2.2 (the latest one still supported by GitHub MacOS x86_64 runners) is
# dropped, or if MacOS is removed from the test matrix, see https://github.com/huggingface/peft/issues/2431.
# Note: the function name is fixed by the pytest plugin system, don't change it
@pytest.hookimpl(hookwrapper=True)
def pytest_runtest_makereport(item, call):
"""
Plug into the pytest test report generation to skip a specific MacOS failure caused by transformers.
The error was introduced by https://github.com/huggingface/transformers/pull/37785, which results in torch.load
failing when using torch < 2.6.
Since the MacOS x86 runners need to use an older torch version, those steps are necessary to get the CI green.
"""
outcome = yield
rep = outcome.get_result()
# ref:
# https://github.com/huggingface/transformers/blob/858ce6879a4aa7fa76a7c4e2ac20388e087ace26/src/transformers/utils/import_utils.py#L1418
error_msg = re.compile(r"Due to a serious vulnerability issue in `torch.load`")
# notes:
# - pytest uses hard-coded strings, we cannot import and use constants
# https://docs.pytest.org/en/stable/reference/reference.html#pytest.TestReport
# - errors can happen during call (running the test) but also setup (e.g. in fixtures)
if rep.failed and (rep.when in ("setup", "call")) and (platform.system() == "Darwin"):
exc_msg = str(call.excinfo.value)
if error_msg.search(exc_msg):
# turn this failure into an xfail:
rep.outcome = "skipped"
# for this attribute, see:
# https://github.com/pytest-dev/pytest/blob/bd6877e5874b50ee57d0f63b342a67298ee9a1c3/src/_pytest/reports.py#L266C5-L266C13
rep.wasxfail = "Error known to occur on MacOS with older torch versions, won't be fixed"
|