[Fix, Add] fix bug with metric names

app.py

@@ -118,7 +118,7 @@ with gr.Blocks(title="LLM Similarity Analyzer") as demo:
     model_dropdown = gr.Dropdown(
         choices=get_leaderboard_models_cached(),
         label="Select Models",
-        value=["HuggingFaceTB/SmolLM2-1.7B-Instruct", "meta_llama/Llama-3.2-3B-Instruct", "Qwen/Qwen2.5-72B-Instruct"],
+        value=["HuggingFaceTB/SmolLM2-1.7B-Instruct", "tiiuae/Falcon3-7B-Instruct", "google/gemma-2-27b-it", "Qwen/Qwen2.5-72B-Instruct"],
         multiselect=True,
         filterable=True,
         allow_custom_value=False,
@@ -158,6 +158,7 @@ with gr.Blocks(title="LLM Similarity Analyzer") as demo:
 - **Models**: Open LLM Leaderboard models \n
     - Every model evaluation is gated on Hugging Face and access has to be requested. \n
     - We requested access for the most popular models, but some may be missing. \n
+    - Notably, loading data is not possible for many meta-llama and gemma models.
 - **Metrics**: CAPA (probabilistic), CAPA (deterministic), Error Consistency""")
 
 if __name__ == "__main__":

src/similarity.py

@@ -31,9 +31,9 @@ def compute_similarity(metric: Metrics, outputs_a: list[np.array], outputs_b: li
 
 def compute_pairwise_similarities(metric_name: str, probs: list[list[np.array]], gts: list[list[int]]) -> np.array:
     # Select chosen metric
-    if metric_name == "Kappa_p (prob.)":
+    if metric_name == "CAPA":
         metric = CAPA()
-    elif metric_name == "Kappa_p (det.)":
+    elif metric_name == "CAPA (det.)":
         metric = CAPA(prob=False)
         # Convert probabilities to one-hot
         probs = [[one_hot(p) for p in model_probs] for model_probs in probs]
@@ -51,7 +51,7 @@ def compute_pairwise_similarities(metric_name: str, probs: list[list[np.array]],
             gt_b = gts[j]
 
             # Format softmax outputs
-            if metric_name == "Kappa_p (prob.)":
+            if metric_name == "CAPA":
                 outputs_a = [softmax(logits) for logits in outputs_a]
                 outputs_b = [softmax(logits) for logits in outputs_b]