test

duckdb-nsql/eval/evaluate.py

@@ -114,9 +114,11 @@ def compute_exact_match_metric(
     return exact_match
 
 
-def evaluate_with_timeout(evaluator, *args, timeout):
+def evaluate_with_timeout(evaluator, gold_db, reference, prediction,
+                          setup_sql, validate_sql, turn_scores, idx, category, timeout):
     with ThreadPoolExecutor(max_workers=1) as executor:
-        future = executor.submit(evaluator.evaluate_one, *args)
+        future = executor.submit(evaluator.evaluate_one, gold_db, reference, prediction,
+                                 setup_sql, validate_sql, turn_scores, idx=idx, category=category)
         try:
             result = future.result(timeout=timeout)
         except TimeoutError:
@@ -150,15 +152,12 @@ def compute_test_suite_metric(
             zip(predictions, references, gold_dbs, setup_sqls, validate_sqls, categories),
             total=len(predictions),
     ):
-        turn_idx = 0
-        # skip final utterance-query pairs
-        if turn_idx < 0:
-            continue
+        turn_idx = 0  # or any value that represents the current index if this is incorrect
 
         # Use the new function to evaluate with timeout
         ex_metrics = evaluate_with_timeout(
             evaluator, gold_db, reference, prediction, setup_sql, validate_sql,
-            turn_scores, timeout=TIMEOUT_SECONDS
+            turn_scores, idx=turn_idx, category=category, timeout=TIMEOUT_SECONDS
         )
 
         if ex_metrics: