Add cross-linking of QA models

app.py

@@ -67,6 +67,7 @@ def get_supported_metrics():
         # in the same environment. Refactor to avoid needing to actually load
         # the metric.
         try:
+            print(f"INFO -- Attempting to load metric: {metric}")
             metric_func = load(metric)
         except Exception as e:
             print(e)
@@ -103,7 +104,7 @@ st.markdown(
     Welcome to Hugging Face's automatic model evaluator! This application allows
     you to evaluate 🤗 Transformers
     [models](https://huggingface.co/models?library=transformers&sort=downloads)
-    across a wide variety of datasets on the Hub -- all for free! Please select
+    across a wide variety of datasets on the Hub. Please select
     the dataset and configuration below. The results of your evaluation will be
     displayed on the [public
     leaderboard](https://huggingface.co/spaces/autoevaluate/leaderboards).
@@ -345,8 +346,12 @@ with st.expander("Advanced configuration"):
     )
 
 with st.form(key="form"):
+    # Grab all models fine-tuned on SQuAD for question answering tasks
+    if selected_task == "extractive_question_answering":
+        compatible_models = get_compatible_models(selected_task, [selected_dataset, "squad", "squad_v2"])
+    else:
+        compatible_models = get_compatible_models(selected_task, [selected_dataset])
 
-    compatible_models = get_compatible_models(selected_task, selected_dataset)
     selected_models = st.multiselect(
         "Select the models you wish to evaluate",
         compatible_models,

utils.py

@@ -1,4 +1,4 @@
-from typing import Dict, Union
+from typing import Dict, List, Union
 
 import jsonlines
 import requests
@@ -19,9 +19,6 @@ HUB_TASK_TO_AUTOTRAIN_TASK = {v: k for k, v in AUTOTRAIN_TASK_TO_HUB_TASK.items(
 LOGS_REPO = "evaluation-job-logs"
 
 
-api = HfApi()
-
-
 def get_auth_headers(token: str, prefix: str = "autonlp"):
     return {"Authorization": f"{prefix} {token}"}
 
@@ -65,14 +62,26 @@ def get_metadata(dataset_name: str) -> Union[Dict, None]:
         return None
 
 
-def get_compatible_models(task, dataset_name):
-    # TODO: relax filter on PyTorch models once supported in AutoTrain
-    filt = ModelFilter(
-        task=AUTOTRAIN_TASK_TO_HUB_TASK[task],
-        trained_dataset=dataset_name,
-        library=["transformers", "pytorch"],
-    )
-    compatible_models = api.list_models(filter=filt)
+def get_compatible_models(task: str, dataset_ids: List[str]) -> List[str]:
+    """
+    Returns all model IDs that are compatible with the given task and dataset names.
+
+    Args:
+        task (`str`): The task to search for.
+        dataset_names (`List[str]`): A list of dataset names to search for.
+
+    Returns:
+        A list of model IDs, sorted alphabetically.
+    """
+    # TODO: relax filter on PyTorch models if TensorFlow supported in AutoTrain
+    compatible_models = []
+    for dataset_id in dataset_ids:
+        model_filter = ModelFilter(
+            task=AUTOTRAIN_TASK_TO_HUB_TASK[task],
+            trained_dataset=dataset_id,
+            library=["transformers", "pytorch"],
+        )
+        compatible_models.extend(HfApi().list_models(filter=model_filter))
     return sorted([model.modelId for model in compatible_models])