Correcting significance in `tournament_results` with FDR correction method

server.py

@@ -167,6 +167,30 @@ def check_significance(model_a_path, model_b_path):
     result = check_significance_wait_for_result(result_url)
     return result
 
+def correct_pvals_for_fdr(model_task_pvals, fdr_alpha=0.05, repeat_on_conn_timeout=10):
+    url = 'https://czechllm.fit.vutbr.cz/benczechmark-leaderboard/compare_significance/correct_pvals_for_fdr'
+
+    data = {
+        "pvals": model_task_pvals,
+        "fdr_alpha": fdr_alpha
+    }
+
+    # prepare and send request
+    response = check_significance_repeat_on_conn_timeout(
+        repeat_on_conn_timeout,
+        requests.post, url, json=data, timeout=60 * 5
+    )
+
+    # check response
+    if response.status_code == 200:
+        result = response.json()
+    elif response.status_code == 429:
+        raise CheckSignificanceError('Server is too busy. Please try again later.')
+    else:
+        raise CheckSignificanceError(f'Failed to submit task. Status code: {response.status_code}')
+
+    return result
+
 class NoneLock:
     def __init__(self, *args, **kwargs):
         pass
@@ -543,6 +567,7 @@ class LeaderboardServer:
         
         with self.var_lock.ro:
             tournament_results = pre_submit.tournament_results if pre_submit else self.tournament_results
+            tournament_results = self._correct_significance_in_tournament_results(tournament_results)
             
             for competitor_id in tournament_results[submission_id].keys() - {submission_id}: # without self
                 if competitor_id not in self.submission_id_to_data:
@@ -560,7 +585,7 @@ class LeaderboardServer:
                         if to_csv:
                             match_results[task] = tournament_results[submission_id][competitor_id][task]["significant"]
                         else:
-                            match_task_result_details = dict.fromkeys(["significant", "p_value"])  # order has impact to sorting DataFrame
+                            match_task_result_details = dict.fromkeys(["significant", "corrected_p_value", "p_value"])  # order has impact to sorting DataFrame
                             match_task_result_details.update(copy.deepcopy(tournament_results[submission_id][competitor_id][task]))
                             match_task_result_details["significant"] = str(match_task_result_details["significant"]).lower()  # originaly bool
                             match_task_result_significant = match_task_result_details["significant"]
@@ -611,6 +636,20 @@ class LeaderboardServer:
                 dataframe = dataframe.style.apply(self._model_tournament_table_highlight_true_and_false, axis=None)
             return dataframe
 
+    def _correct_significance_in_tournament_results(self, tournament_results, alpha=0.05):
+        tournament_results = copy.deepcopy(tournament_results)
+        
+        for submission_id in tournament_results:
+            for task in self.TASKS_METADATA:
+                competitors = [competitor_id for competitor_id in tournament_results[submission_id].keys() - {submission_id}]  # without self
+                model_task_pvals = [tournament_results[submission_id][competitor_id][task]["p_value"] for competitor_id in competitors]
+                corrected_model_task_pvals = correct_pvals_for_fdr(model_task_pvals)
+                for competitor_id, task_pval in zip(competitors, corrected_model_task_pvals):
+                    tournament_results[submission_id][competitor_id][task]["corrected_p_value"] = task_pval
+                    tournament_results[submission_id][competitor_id][task]["significant"] = bool(task_pval < alpha)
+        
+        return tournament_results
+
     def _dataframe_to_csv(self, dataframe, filename):
         try:
             if not os.path.isdir(self.DIR_DATAFRAMES_CSV):
@@ -689,6 +728,7 @@ class LeaderboardServer:
     def _get_leaderboard(self, pre_submit=None, category=None, to_csv=False):
         with self.var_lock.ro:
             tournament_results = pre_submit.tournament_results if pre_submit else self.tournament_results
+            tournament_results = self._correct_significance_in_tournament_results(tournament_results)
             category = category if category else self.TASKS_CATEGORY_OVERALL
 
             if len(tournament_results) == 0: