Update main.py

main.py

@@ -81,20 +81,23 @@ def load_questions():
         task_id = item.get('task_id')
         question_text = item.get('Question')
         final_answer = item.get('Final answer')
-
+        
+        # Validate required fields
         if task_id and question_text and final_answer is not None:
-            questions_for_api.append({
-                "task_id": str(task_id), # Ensure ID is string
-                "question": question_text
-            })
-            ground_truth_answers[str(task_id)] = str(final_answer) # Ensure answer is string
+            # Create a copy of the item and remove fields we don't want
+            processed_item = item.copy()
+            processed_item.pop('Final answer', None)  # Remove Final answer
+            processed_item.pop('Annotator Annotation', None)  # Remove Annotator Annotation
+            
+            # Store in questions_for_api
+            questions_for_api.append(processed_item)
+            
+            # Still store the ground truth answers separately
+            ground_truth_answers[str(task_id)] = str(final_answer)
             processed_count += 1
         else:
             logger.warning(f"Skipping item due to missing fields (task_id, Question, or Final answer): {item}")
 
-    logger.info(f"Successfully processed and loaded {processed_count} questions for the API.")
-    # print(questions_for_api) # Keep this commented unless debugging
-
     if not questions_for_api:
          logger.error("CRITICAL: No valid questions loaded after filtering. API endpoints needing questions will fail.")
          # Depending on requirements, you might want to exit or raise an error here