use multi processing only for 500 or more samples

VerifiableRewardsForScalableLogicalReasoning.py

@@ -314,18 +314,21 @@ class VerifiableRewardsForScalableLogicalReasoning(evaluate.Metric):
 
             eval_inputs.append((prediction, validation_program, eval_config))
 
-        # Process evaluations in parallel
-        num_cpus = max(1, mp.cpu_count() - 1)  # Leave one CPU free
-        with mp.Pool(processes=num_cpus) as pool:
-            results = list(tqdm(
-                pool.starmap(_evaluate_with_prolog, eval_inputs),
-                total=len(eval_inputs),
-                desc="Evaluating rules (parallel)"
-            ))
-        # no multiprocessing in the main thread, so we can use tqdm directly
-        # results = []
-        # for prediction, validation_program, eval_config in tqdm(eval_inputs, total=len(predictions), desc="Evaluating rules"):
-        #     results.append(_evaluate_with_prolog(prediction, validation_program, eval_config))
+        # if more than 1k predictions, we use multiprocessing to speed up the evaluation
+        if len(eval_inputs) > 1000:
+            # Process evaluations in parallel
+            num_cpus = max(1, mp.cpu_count() - 1)  # Leave one CPU free
+            with mp.Pool(processes=num_cpus) as pool:
+                results = list(tqdm(
+                    pool.starmap(_evaluate_with_prolog, eval_inputs),
+                    total=len(eval_inputs),
+                    desc="Evaluating rules (parallel)"
+                ))
+        else:
+            # Evaluate in the main thread (no multiprocessing)
+            results = []
+            for prediction, validation_program, eval_config in tqdm(eval_inputs, total=len(predictions), desc="Evaluating rules"):
+                results.append(_evaluate_with_prolog(prediction, validation_program, eval_config))
 
         # Calculate metrics
         partial_scores = [result["partial_score"] for result in results]