Started to run evaluations outside of submit loop

eval/create_dataset.py

@@ -0,0 +1,23 @@
+import pandas as pd
+import phoenix as px
+from phoenix.client import Client
+
+
+def create_dataset():
+    dataset_df = pd.read_json("./data/metadata.jsonl", lines=True)
+
+    # Script should be run with a running phoenix server, if not uncomment:
+    # _ = px.launch_app()
+    px_client = Client()
+    dataset = px_client.datasets.create_dataset(
+        dataframe=dataset_df,
+        name="gaia",
+        input_keys=["Question"],
+        output_keys=["Final answer"],
+        metadata_keys=["task_id", "Annotator Metadata", "file_name"],
+    )
+    print(f"Dataset created: {dataset.id}")
+
+
+if __name__ == "__main__":
+    create_dataset()

eval/eval_notebook.ipynb

@@ -0,0 +1,135 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 19,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import pandas as pd\n",
+    "import json\n",
+    "from phoenix.client import Client\n",
+    "\n",
+    "# Load the existing spans\n",
+    "spans_df = Client().spans.get_spans_dataframe(project_name=\"default\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 20,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Load the source of truth\n",
+    "dataset_df = pd.read_json(\"../data/metadata.jsonl\", lines=True)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 21,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Filter by root agents\n",
+    "agents_df = spans_df[(spans_df.span_kind == 'AGENT') & (spans_df.parent_id.isna()) & (spans_df.status_code == 'OK')]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 22,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "/var/folders/pj/v1zrqj1d10x9_1rd2njh_r_r0000gn/T/ipykernel_98186/3107371246.py:2: SettingWithCopyWarning: \n",
+      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
+      "Try using .loc[row_indexer,col_indexer] = value instead\n",
+      "\n",
+      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
+      "  agents_df[\"task\"] = agents_df[\"attributes.input.value\"].apply(json.loads).apply(lambda x : x[\"task\"]).str.replace(r'\\s*The mentionned file can be downloaded from.*$', '', regex=True)\n"
+     ]
+    }
+   ],
+   "source": [
+    "# Retrieve the right question and add the answer\n",
+    "agents_df[\"task\"] = agents_df[\"attributes.input.value\"].apply(json.loads).apply(lambda x : x[\"task\"]).str.replace(r'\\s*The mentionned file can be downloaded from.*$', '', regex=True)\n",
+    "agents_merged_df = pd.merge(agents_df,dataset_df,how=\"left\",left_on=\"task\", right_on=\"Question\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 29,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from phoenix.evals.evaluators import bind_evaluator, async_evaluate_dataframe\n",
+    "from phoenix.evals.metrics import exact_match\n",
+    "from evaluators import conciseness_evaluator\n",
+    "\n",
+    "# Define the evaluator\n",
+    "exact_match_eval = bind_evaluator(evaluator=exact_match, input_mapping= { \"output\": \"attributes.output.value\", \"expected\": \"Final answer\"})\n",
+    "conciseness_evaluator = bind_evaluator(evaluator=conciseness_evaluator, input_mapping={ \"output\": \"attributes.output.value\", \"expected\": \"Final answer\"})\n",
+    "results_df = await async_evaluate_dataframe(agents_merged_df, evaluators=[exact_match_eval, conciseness_evaluator])\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 34,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "results_df[\"exact_match\"] = results_df.exact_match_score.apply(json.loads).apply(lambda x : x[\"score\"])\n",
+    "results_df[\"conciseness\"] = results_df.conciseness_evaluator_score.apply(json.loads).apply(lambda x : x[\"label\"])\n",
+    "results_df[\"agent_type\"] = results_df[\"attributes.smolagents\"].apply(lambda x : \"multi_agent\" if \"managed_agents\" in x else \"llm_agent\")\n",
+    "results_filtered_df = results_df[[\"name\", \"span_kind\", \"start_time\", \"context.span_id\", \"context.trace_id\",\"attributes.output.value\", \"task_id\", \"Question\", \"Final answer\", \"agent_type\", \"exact_match_score\", \"conciseness_evaluator_score\", \"exact_match\", \"conciseness\"]]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 38,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "/Users/romainfayoux/Documents/Programmation/Final_Assignment_Template/.venv/lib/python3.12/site-packages/phoenix/evals/utils.py:367: FutureWarning: The behavior of DataFrame concatenation with empty or all-NA entries is deprecated. In a future version, this will no longer exclude empty or all-NA columns when determining the result dtypes. To retain the old behavior, exclude the relevant entries before the concat operation.\n",
+      "  result_df = pd.concat(result_dfs, ignore_index=True)\n"
+     ]
+    }
+   ],
+   "source": [
+    "# Upload results\n",
+    "import numpy as np\n",
+    "from phoenix.evals.utils import to_annotation_dataframe\n",
+    "\n",
+    "annotation_df = to_annotation_dataframe(results_filtered_df)\n",
+    "annotation_df = annotation_df.replace({np.nan: None})\n",
+    "Client().spans.log_span_annotations_dataframe(dataframe=annotation_df)\n"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Final_Assignment_Template",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.12.11"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

eval/evaluators.py

@@ -0,0 +1,11 @@
+from phoenix.evals import create_evaluator
+
+@create_evaluator(name="conciseness_evaluator")
+def conciseness_evaluator(output: str, expected: str):
+    ratio = (len(output) / len(expected))
+    if ratio < 0.5:
+        return "too short"
+    elif ratio > 3.0:
+        return "too long"
+    else:
+        return "concise"