Hugging Face's logo

Spaces:
autoevaluate
/
model-evaluator
Runtime error

App Files Community
59
model-evaluator / app.py
lewtun's picture
lewtun HF Staff
Archive project
daea199
raw
history blame
30.2 kB
 import os
import time
from pathlib import Path
import pandas as pd
import streamlit as st
import yaml
from datasets import get_dataset_config_names
from dotenv import load_dotenv
from huggingface_hub import list_datasets
from evaluation import filter_evaluated_models
from utils import (
AUTOTRAIN_TASK_TO_HUB_TASK,
commit_evaluation_log,
create_autotrain_project_name,
format_col_mapping,
get_compatible_models,
get_config_metadata,
get_dataset_card_url,
get_key,
get_metadata,
http_get,
http_post,
)
if Path(".env").is_file():
load_dotenv(".env")
HF_TOKEN = os.getenv("HF_TOKEN")
AUTOTRAIN_USERNAME = os.getenv("AUTOTRAIN_USERNAME")
AUTOTRAIN_BACKEND_API = os.getenv("AUTOTRAIN_BACKEND_API")
DATASETS_PREVIEW_API = os.getenv("DATASETS_PREVIEW_API")
# Put image tasks on top
TASK_TO_ID = {
"image_binary_classification": 17,
"image_multi_class_classification": 18,
"binary_classification": 1,
"multi_class_classification": 2,
"natural_language_inference": 22,
"entity_extraction": 4,
"extractive_question_answering": 5,
"translation": 6,
"summarization": 8,
"text_zero_shot_classification": 23,
}
TASK_TO_DEFAULT_METRICS = {
"binary_classification": ["f1", "precision", "recall", "auc", "accuracy"],
"multi_class_classification": [
"f1",
"precision",
"recall",
"accuracy",
],
"natural_language_inference": ["f1", "precision", "recall", "auc", "accuracy"],
"entity_extraction": ["precision", "recall", "f1", "accuracy"],
"extractive_question_answering": ["f1", "exact_match"],
"translation": ["sacrebleu"],
"summarization": ["rouge1", "rouge2", "rougeL", "rougeLsum"],
"image_binary_classification": ["f1", "precision", "recall", "auc", "accuracy"],
"image_multi_class_classification": [
"f1",
"precision",
"recall",
"accuracy",
],
"text_zero_shot_classification": ["accuracy", "loss"],
}
AUTOTRAIN_TASK_TO_LANG = {
"translation": "en2de",
"image_binary_classification": "unk",
"image_multi_class_classification": "unk",
}
AUTOTRAIN_MACHINE = {"text_zero_shot_classification": "r5.16x"}
SUPPORTED_TASKS = list(TASK_TO_ID.keys())
# Extracted from utils.get_supported_metrics
# Hardcoded for now due to speed / caching constraints
SUPPORTED_METRICS = [
"accuracy",
"bertscore",
"bleu",
"cer",
"chrf",
"code_eval",
"comet",
"competition_math",
"coval",
"cuad",
"exact_match",
"f1",
"frugalscore",
"google_bleu",
"mae",
"mahalanobis",
"matthews_correlation",
"mean_iou",
"meteor",
"mse",
"pearsonr",
"perplexity",
"precision",
"recall",
"roc_auc",
"rouge",
"sacrebleu",
"sari",
"seqeval",
"spearmanr",
"squad",
"squad_v2",
"ter",
"trec_eval",
"wer",
"wiki_split",
"xnli",
"angelina-wang/directional_bias_amplification",
"jordyvl/ece",
"lvwerra/ai4code",
"lvwerra/amex",
]
#######
# APP #
#######
st.title("Evaluation on the Hub")
st.warning(
"**⚠️ This project has been archived. If you want to evaluate LLMs, checkout [this collection](https://huggingface.co/collections/clefourrier/llm-leaderboards-and-benchmarks-✨-64f99d2e11e92ca5568a7cce) of leaderboards.**"
)
st.markdown(
"""
Welcome to Hugging Face's automatic model evaluator 👋!
This application allows you to evaluate 🤗 Transformers
[models](https://huggingface.co/models?library=transformers&sort=downloads)
across a wide variety of [datasets](https://huggingface.co/datasets) on the
Hub. Please select the dataset and configuration below. The results of your
evaluation will be displayed on the [public
leaderboards](https://huggingface.co/spaces/autoevaluate/leaderboards). For
more details, check out out our [blog
post](https://huggingface.co/blog/eval-on-the-hub).
"""
)
# all_datasets = [d.id for d in list_datasets()]
# query_params = st.experimental_get_query_params()
# if "first_query_params" not in st.session_state:
# st.session_state.first_query_params = query_params
# first_query_params = st.session_state.first_query_params
# default_dataset = all_datasets[0]
# if "dataset" in first_query_params:
# if len(first_query_params["dataset"]) > 0 and first_query_params["dataset"][0] in all_datasets:
# default_dataset = first_query_params["dataset"][0]
# selected_dataset = st.selectbox(
# "Select a dataset",
# all_datasets,
# index=all_datasets.index(default_dataset),
# help="""Datasets with metadata can be evaluated with 1-click. Configure an evaluation job to add \
# new metadata to a dataset card.""",
# )
# st.experimental_set_query_params(**{"dataset": [selected_dataset]})
# # Check if selected dataset can be streamed
# is_valid_dataset = http_get(
# path="/is-valid",
# domain=DATASETS_PREVIEW_API,
# params={"dataset": selected_dataset},
# ).json()
# if is_valid_dataset["viewer"] is False and is_valid_dataset["preview"] is False:
# st.error(
# """The dataset you selected is not currently supported. Open a \
# [discussion](https://huggingface.co/spaces/autoevaluate/model-evaluator/discussions) for support."""
# )
# metadata = get_metadata(selected_dataset, token=HF_TOKEN)
# print(f"INFO -- Dataset metadata: {metadata}")
# if metadata is None:
# st.warning("No evaluation metadata found. Please configure the evaluation job below.")
# with st.expander("Advanced configuration"):
# # Select task
# selected_task = st.selectbox(
# "Select a task",
# SUPPORTED_TASKS,
# index=SUPPORTED_TASKS.index(metadata[0]["task_id"]) if metadata is not None else 0,
# help="""Don't see your favourite task here? Open a \
# [discussion](https://huggingface.co/spaces/autoevaluate/model-evaluator/discussions) to request it!""",
# )
# # Select config
# configs = get_dataset_config_names(selected_dataset)
# selected_config = st.selectbox(
# "Select a config",
# configs,
# help="""Some datasets contain several sub-datasets, known as _configurations_. \
# Select one to evaluate your models on. \
# See the [docs](https://huggingface.co/docs/datasets/master/en/load_hub#configurations) for more details.
# """,
# )
# # Some datasets have multiple metadata (one per config), so we grab the one associated with the selected config
# config_metadata = get_config_metadata(selected_config, metadata)
# print(f"INFO -- Config metadata: {config_metadata}")
# # Select splits
# splits_resp = http_get(
# path="/splits",
# domain=DATASETS_PREVIEW_API,
# params={"dataset": selected_dataset},
# )
# if splits_resp.status_code == 200:
# split_names = []
# all_splits = splits_resp.json()
# for split in all_splits["splits"]:
# if split["config"] == selected_config:
# split_names.append(split["split"])
# if config_metadata is not None:
# eval_split = config_metadata["splits"].get("eval_split", None)
# else:
# eval_split = None
# selected_split = st.selectbox(
# "Select a split",
# split_names,
# index=split_names.index(eval_split) if eval_split is not None else 0,
# help="Be wary when evaluating models on the `train` split.",
# )
# # Select columns
# rows_resp = http_get(
# path="/first-rows",
# domain=DATASETS_PREVIEW_API,
# params={
# "dataset": selected_dataset,
# "config": selected_config,
# "split": selected_split,
# },
# ).json()
# col_names = list(pd.json_normalize(rows_resp["rows"][0]["row"]).columns)
# st.markdown("**Map your dataset columns**")
# st.markdown(
# """The model evaluator uses a standardised set of column names for the input examples and labels. \
# Please define the mapping between your dataset columns (right) and the standardised column names (left)."""
# )
# col1, col2 = st.columns(2)
# # TODO: find a better way to layout these items
# # TODO: need graceful way of handling dataset <--> task mismatch for datasets with metadata
# col_mapping = {}
# if selected_task in ["binary_classification", "multi_class_classification"]:
# with col1:
# st.markdown("`text` column")
# st.text("")
# st.text("")
# st.text("")
# st.text("")
# st.markdown("`target` column")
# with col2:
# text_col = st.selectbox(
# "This column should contain the text to be classified",
# col_names,
# index=col_names.index(get_key(config_metadata["col_mapping"], "text"))
# if config_metadata is not None
# else 0,
# )
# target_col = st.selectbox(
# "This column should contain the labels associated with the text",
# col_names,
# index=col_names.index(get_key(config_metadata["col_mapping"], "target"))
# if config_metadata is not None
# else 0,
# )
# col_mapping[text_col] = "text"
# col_mapping[target_col] = "target"
# elif selected_task == "text_zero_shot_classification":
# with col1:
# st.markdown("`text` column")
# st.text("")
# st.text("")
# st.text("")
# st.text("")
# st.markdown("`classes` column")
# st.text("")
# st.text("")
# st.text("")
# st.text("")
# st.markdown("`target` column")
# with col2:
# text_col = st.selectbox(
# "This column should contain the text to be classified",
# col_names,
# index=col_names.index(get_key(config_metadata["col_mapping"], "text"))
# if config_metadata is not None
# else 0,
# )
# classes_col = st.selectbox(
# "This column should contain the classes associated with the text",
# col_names,
# index=col_names.index(get_key(config_metadata["col_mapping"], "classes"))
# if config_metadata is not None
# else 0,
# )
# target_col = st.selectbox(
# "This column should contain the index of the correct class",
# col_names,
# index=col_names.index(get_key(config_metadata["col_mapping"], "target"))
# if config_metadata is not None
# else 0,
# )
# col_mapping[text_col] = "text"
# col_mapping[classes_col] = "classes"
# col_mapping[target_col] = "target"
# if selected_task in ["natural_language_inference"]:
# config_metadata = get_config_metadata(selected_config, metadata)
# with col1:
# st.markdown("`text1` column")
# st.text("")
# st.text("")
# st.text("")
# st.text("")
# st.text("")
# st.markdown("`text2` column")
# st.text("")
# st.text("")
# st.text("")
# st.text("")
# st.text("")
# st.markdown("`target` column")
# with col2:
# text1_col = st.selectbox(
# "This column should contain the first text passage to be classified",
# col_names,
# index=col_names.index(get_key(config_metadata["col_mapping"], "text1"))
# if config_metadata is not None
# else 0,
# )
# text2_col = st.selectbox(
# "This column should contain the second text passage to be classified",
# col_names,
# index=col_names.index(get_key(config_metadata["col_mapping"], "text2"))
# if config_metadata is not None
# else 0,
# )
# target_col = st.selectbox(
# "This column should contain the labels associated with the text",
# col_names,
# index=col_names.index(get_key(config_metadata["col_mapping"], "target"))
# if config_metadata is not None
# else 0,
# )
# col_mapping[text1_col] = "text1"
# col_mapping[text2_col] = "text2"
# col_mapping[target_col] = "target"
# elif selected_task == "entity_extraction":
# with col1:
# st.markdown("`tokens` column")
# st.text("")
# st.text("")
# st.text("")
# st.text("")
# st.markdown("`tags` column")
# with col2:
# tokens_col = st.selectbox(
# "This column should contain the array of tokens to be classified",
# col_names,
# index=col_names.index(get_key(config_metadata["col_mapping"], "tokens"))
# if config_metadata is not None
# else 0,
# )
# tags_col = st.selectbox(
# "This column should contain the labels associated with each part of the text",
# col_names,
# index=col_names.index(get_key(config_metadata["col_mapping"], "tags"))
# if config_metadata is not None
# else 0,
# )
# col_mapping[tokens_col] = "tokens"
# col_mapping[tags_col] = "tags"
# elif selected_task == "translation":
# with col1:
# st.markdown("`source` column")
# st.text("")
# st.text("")
# st.text("")
# st.text("")
# st.markdown("`target` column")
# with col2:
# text_col = st.selectbox(
# "This column should contain the text to be translated",
# col_names,
# index=col_names.index(get_key(config_metadata["col_mapping"], "source"))
# if config_metadata is not None
# else 0,
# )
# target_col = st.selectbox(
# "This column should contain the target translation",
# col_names,
# index=col_names.index(get_key(config_metadata["col_mapping"], "target"))
# if config_metadata is not None
# else 0,
# )
# col_mapping[text_col] = "source"
# col_mapping[target_col] = "target"
# elif selected_task == "summarization":
# with col1:
# st.markdown("`text` column")
# st.text("")
# st.text("")
# st.text("")
# st.text("")
# st.markdown("`target` column")
# with col2:
# text_col = st.selectbox(
# "This column should contain the text to be summarized",
# col_names,
# index=col_names.index(get_key(config_metadata["col_mapping"], "text"))
# if config_metadata is not None
# else 0,
# )
# target_col = st.selectbox(
# "This column should contain the target summary",
# col_names,
# index=col_names.index(get_key(config_metadata["col_mapping"], "target"))
# if config_metadata is not None
# else 0,
# )
# col_mapping[text_col] = "text"
# col_mapping[target_col] = "target"
# elif selected_task == "extractive_question_answering":
# if config_metadata is not None:
# col_mapping = config_metadata["col_mapping"]
# # Hub YAML parser converts periods to hyphens, so we remap them here
# col_mapping = format_col_mapping(col_mapping)
# with col1:
# st.markdown("`context` column")
# st.text("")
# st.text("")
# st.text("")
# st.text("")
# st.markdown("`question` column")
# st.text("")
# st.text("")
# st.text("")
# st.text("")
# st.markdown("`answers.text` column")
# st.text("")
# st.text("")
# st.text("")
# st.text("")
# st.markdown("`answers.answer_start` column")
# with col2:
# context_col = st.selectbox(
# "This column should contain the question's context",
# col_names,
# index=col_names.index(get_key(col_mapping, "context")) if config_metadata is not None else 0,
# )
# question_col = st.selectbox(
# "This column should contain the question to be answered, given the context",
# col_names,
# index=col_names.index(get_key(col_mapping, "question")) if config_metadata is not None else 0,
# )
# answers_text_col = st.selectbox(
# "This column should contain example answers to the question, extracted from the context",
# col_names,
# index=col_names.index(get_key(col_mapping, "answers.text")) if config_metadata is not None else 0,
# )
# answers_start_col = st.selectbox(
# "This column should contain the indices in the context of the first character of each `answers.text`",
# col_names,
# index=col_names.index(get_key(col_mapping, "answers.answer_start"))
# if config_metadata is not None
# else 0,
# )
# col_mapping[context_col] = "context"
# col_mapping[question_col] = "question"
# col_mapping[answers_text_col] = "answers.text"
# col_mapping[answers_start_col] = "answers.answer_start"
# elif selected_task in ["image_binary_classification", "image_multi_class_classification"]:
# with col1:
# st.markdown("`image` column")
# st.text("")
# st.text("")
# st.text("")
# st.text("")
# st.markdown("`target` column")
# with col2:
# image_col = st.selectbox(
# "This column should contain the images to be classified",
# col_names,
# index=col_names.index(get_key(config_metadata["col_mapping"], "image"))
# if config_metadata is not None
# else 0,
# )
# target_col = st.selectbox(
# "This column should contain the labels associated with the images",
# col_names,
# index=col_names.index(get_key(config_metadata["col_mapping"], "target"))
# if config_metadata is not None
# else 0,
# )
# col_mapping[image_col] = "image"
# col_mapping[target_col] = "target"
# # Select metrics
# st.markdown("**Select metrics**")
# st.markdown("The following metrics will be computed")
# html_string = " ".join(
# [
# '<div style="padding-right:5px;padding-left:5px;padding-top:5px;padding-bottom:5px;float:left">'
# + '<div style="background-color:#D3D3D3;border-radius:5px;display:inline-block;padding-right:5px;'
# + 'padding-left:5px;color:white">'
# + metric
# + "</div></div>"
# for metric in TASK_TO_DEFAULT_METRICS[selected_task]
# ]
# )
# st.markdown(html_string, unsafe_allow_html=True)
# selected_metrics = st.multiselect(
# "(Optional) Select additional metrics",
# sorted(list(set(SUPPORTED_METRICS) - set(TASK_TO_DEFAULT_METRICS[selected_task]))),
# help="""User-selected metrics will be computed with their default arguments. \
# For example, `f1` will report results for binary labels. \
# Check out the [available metrics](https://huggingface.co/metrics) for more details.""",
# )
# with st.form(key="form"):
# compatible_models = get_compatible_models(selected_task, [selected_dataset])
# selected_models = st.multiselect(
# "Select the models you wish to evaluate",
# compatible_models,
# help="""Don't see your favourite model in this list? Add the dataset and task it was trained on to the \
# [model card metadata.](https://huggingface.co/docs/hub/models-cards#model-card-metadata)""",
# )
# print("INFO -- Selected models before filter:", selected_models)
# hf_username = st.text_input("Enter your 🤗 Hub username to be notified when the evaluation is finished")
# submit_button = st.form_submit_button("Evaluate models 🚀")
# if submit_button:
# if len(hf_username) == 0:
# st.warning("No 🤗 Hub username provided! Please enter your username and try again.")
# elif len(selected_models) == 0:
# st.warning("⚠️ No models were selected for evaluation! Please select at least one model and try again.")
# elif len(selected_models) > 10:
# st.warning("Only 10 models can be evaluated at once. Please select fewer models and try again.")
# else:
# # Filter out previously evaluated models
# selected_models = filter_evaluated_models(
# selected_models,
# selected_task,
# selected_dataset,
# selected_config,
# selected_split,
# selected_metrics,
# )
# print("INFO -- Selected models after filter:", selected_models)
# if len(selected_models) > 0:
# project_payload = {
# "username": AUTOTRAIN_USERNAME,
# "proj_name": create_autotrain_project_name(selected_dataset, selected_config),
# "task": TASK_TO_ID[selected_task],
# "config": {
# "language": AUTOTRAIN_TASK_TO_LANG[selected_task]
# if selected_task in AUTOTRAIN_TASK_TO_LANG
# else "en",
# "max_models": 5,
# "instance": {
# "provider": "sagemaker" if selected_task in AUTOTRAIN_MACHINE.keys() else "ovh",
# "instance_type": AUTOTRAIN_MACHINE[selected_task]
# if selected_task in AUTOTRAIN_MACHINE.keys()
# else "p3",
# "max_runtime_seconds": 172800,
# "num_instances": 1,
# "disk_size_gb": 200,
# },
# "evaluation": {
# "metrics": selected_metrics,
# "models": selected_models,
# "hf_username": hf_username,
# },
# },
# }
# print(f"INFO -- Payload: {project_payload}")
# project_json_resp = http_post(
# path="/projects/create",
# payload=project_payload,
# token=HF_TOKEN,
# domain=AUTOTRAIN_BACKEND_API,
# ).json()
# print(f"INFO -- Project creation response: {project_json_resp}")
# if project_json_resp["created"]:
# data_payload = {
# "split": 4, # use "auto" split choice in AutoTrain
# "col_mapping": col_mapping,
# "load_config": {"max_size_bytes": 0, "shuffle": False},
# "dataset_id": selected_dataset,
# "dataset_config": selected_config,
# "dataset_split": selected_split,
# }
# data_json_resp = http_post(
# path=f"/projects/{project_json_resp['id']}/data/dataset",
# payload=data_payload,
# token=HF_TOKEN,
# domain=AUTOTRAIN_BACKEND_API,
# ).json()
# print(f"INFO -- Dataset creation response: {data_json_resp}")
# if data_json_resp["download_status"] == 1:
# train_json_resp = http_post(
# path=f"/projects/{project_json_resp['id']}/data/start_processing",
# token=HF_TOKEN,
# domain=AUTOTRAIN_BACKEND_API,
# ).json()
# # For local development we process and approve projects on-the-fly
# if "localhost" in AUTOTRAIN_BACKEND_API:
# with st.spinner("⏳ Waiting for data processing to complete ..."):
# is_data_processing_success = False
# while is_data_processing_success is not True:
# project_status = http_get(
# path=f"/projects/{project_json_resp['id']}",
# token=HF_TOKEN,
# domain=AUTOTRAIN_BACKEND_API,
# ).json()
# if project_status["status"] == 3:
# is_data_processing_success = True
# time.sleep(10)
# # Approve training job
# train_job_resp = http_post(
# path=f"/projects/{project_json_resp['id']}/start_training",
# token=HF_TOKEN,
# domain=AUTOTRAIN_BACKEND_API,
# ).json()
# st.success("✅ Data processing and project approval complete - go forth and evaluate!")
# else:
# # Prod/staging submissions are evaluated in a cron job via run_evaluation_jobs.py
# print(f"INFO -- AutoTrain job response: {train_json_resp}")
# if train_json_resp["success"]:
# train_eval_index = {
# "train-eval-index": [
# {
# "config": selected_config,
# "task": AUTOTRAIN_TASK_TO_HUB_TASK[selected_task],
# "task_id": selected_task,
# "splits": {"eval_split": selected_split},
# "col_mapping": col_mapping,
# }
# ]
# }
# selected_metadata = yaml.dump(train_eval_index, sort_keys=False)
# dataset_card_url = get_dataset_card_url(selected_dataset)
# st.success("✅ Successfully submitted evaluation job!")
# st.markdown(
# f"""
# Evaluation can take up to 1 hour to complete, so grab a ☕️ or 🍵 while you wait:
# * 🔔 A [Hub pull request](https://huggingface.co/docs/hub/repositories-pull-requests-discussions) with the evaluation results will be opened for each model you selected. Check your email for notifications.
# * 📊 Click [here](https://hf.co/spaces/autoevaluate/leaderboards?dataset={selected_dataset}) to view the results from your submission once the Hub pull request is merged.
# * 🥱 Tired of configuring evaluations? Add the following metadata to the [dataset card]({dataset_card_url}) to enable 1-click evaluations:
# """ # noqa
# )
# st.markdown(
# f"""
# ```yaml
# {selected_metadata}
# """
# )
# print("INFO -- Pushing evaluation job logs to the Hub")
# evaluation_log = {}
# evaluation_log["project_id"] = project_json_resp["id"]
# evaluation_log["autotrain_env"] = (
# "staging" if "staging" in AUTOTRAIN_BACKEND_API else "prod"
# )
# evaluation_log["payload"] = project_payload
# evaluation_log["project_creation_response"] = project_json_resp
# evaluation_log["dataset_creation_response"] = data_json_resp
# evaluation_log["autotrain_job_response"] = train_json_resp
# commit_evaluation_log(evaluation_log, hf_access_token=HF_TOKEN)
# else:
# st.error("🙈 Oh no, there was an error submitting your evaluation job!")
# else:
# st.warning("⚠️ No models left to evaluate! Please select other models and try again.")