new charts

evals/.zeno_cache_country-area/reports.pickle

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:6f72e49e684119b434fba6cbd8369a9ea077f8d05bcba159a8456569de2b7b66
+oid sha256:ebe923fd1e82b53bd95580dff84cfae8e0c37824c57186e218e921a01c397b1c
 size 3880

evals/.zeno_cache_crossword/reports.pickle

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:6c1dc251387b1070c8e8b4ce142f75d26006804580d8c2239a9168913460528b
+oid sha256:4335d3a1a4a7ce47c4c9ee39811c090bdeb3a2d565085b411b1e9a471052d705
 size 6586

evals/.zeno_cache_med-mcqa/PREDISTILLsubject.pickle

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:47ada9af3bfaaa3ea9c77bed6774b474d5cf505676302d443c20ace3db98f7a5
+size 11589

evals/.zeno_cache_med-mcqa/folders.pickle

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ec0a6ccf9debf1c16781445c4b9106080d00478b0559469336db7c7b7b9711c8
-size 5
+oid sha256:c21f58777ece01ce867731803c284f07d64ef69d67712eeca9bf9bda7ac4f38b
+size 29

evals/.zeno_cache_med-mcqa/reports.pickle

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ec0a6ccf9debf1c16781445c4b9106080d00478b0559469336db7c7b7b9711c8
-size 5
+oid sha256:0150a18fcad53c825bfed305a0a05c6ddc9cc8bd5f8cd70e630ed4d273d51aa4
+size 3033

evals/.zeno_cache_med-mcqa/slices.pickle

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f4b521b1fc398b7bf3b6be870e597b077110fc2fb2ef61b52b4a204e3b5a45f0
+size 2295

evals/evals.yaml

@@ -13,6 +13,7 @@
 - med-mcqa:
     results-file: ./medmcqa/med-3.5-turbo.jsonl
     second-results-file: ./medmcqa/med-4.jsonl
+    functions-file: ./medmcqa/med_fns.py
     link: https://github.com/openai/evals/commit/19b2cf9ff96b08af68f5c3b4d2c90184844a4fe6
     description: Multiple choice questions from different medical areas.
 - aba-mrpc:

evals/medmcqa/med_fns.py

@@ -0,0 +1,14 @@
+import re
+
+from zeno import DistillReturn, distill
+
+finder = "Subject:(.*)"
+
+
+@distill
+def subject(df, ops):
+    ret_subjs = []
+    for entry in df[ops.data_column]:
+        ret_subjs.append(re.search(finder, entry[1]["content"]).group(1))
+
+    return DistillReturn(distill_output=ret_subjs)

zeno-evals-hub/main.py

@@ -88,7 +88,6 @@ def command_line():
             config.editable = False
 
         zeno_obj = zeno(config)
-        print(config.metadata["expected"])
         if zeno_obj is None:
             sys.exit(1)
         server = get_server(zeno_obj)