Add Seed-OSS-36B

README.md

@@ -72,8 +72,8 @@ If you are from HPAI, you can add your model onto our shared `.csv` file of resu
 2. Parse the CSV files onto JSON, which is what the Leaderboard will take as ground truth
 
     ```
-    $ uv run results/parse.py results/results_v3_mlcad_icarus.csv # will generate results/results_v3_mlcad_icarus.json
-    $ uv run results/parse.py results/results_v3_mlcad_verilator.csv # will generate results/results_v3_mlcad_verilator.json
+    $ uv run -m results.parse results/results_v3_mlcad_icarus.csv # will generate results/results_v3_mlcad_icarus.json
+    $ uv run -m results.parse results/results_v3_mlcad_verilator.csv # will generate results/results_v3_mlcad_verilator.json
     ```
 
     The application is hardcoded to look for `results_icarus.json` and `results_verilator.json.` Rename the files you just created:

results/aggregated_scores_icarus.csv

@@ -7,6 +7,7 @@ Qwen3 236B A22B,69.16,67.53,74.83,68.36,50.48,60.01
 gpt-oss-120b,70.52,77.82,75.89,78.74,52.82,69.41
 Qwen2.5 72B,49.36,50.41,50.22,50.74,46.51,47.42
 Llama 3.(1-3) 70B,39.48,43.15,39.47,40.83,39.53,64.29
+Seed-OSS-36B,67.52,73.85,69.51,74.32,60.94,69.58
 QwQ 32B,62.6,40.06,65.02,38.68,54.6,52.6
 Qwen2.5 32B,50.39,41.4,50.86,41.01,48.86,45.0
 Magistral Small 2506,40.82,22.62,40.96,19.96,40.37,46.88

results/aggregated_scores_verilator.csv

@@ -7,6 +7,7 @@ Qwen3 236B A22B,69.17,66.8,74.24,67.55,52.44,60.01
 gpt-oss-120b,70.18,74.91,75.45,75.4,52.82,70.46
 Qwen2.5 72B,51.72,52.29,52.7,52.44,48.49,50.95
 Llama 3.(1-3) 70B,40.06,44.71,39.84,42.69,40.79,63.12
+Seed-OSS-36B,67.75,71.92,69.69,72.18,61.35,69.58
 QwQ 32B,63.76,41.14,66.09,38.64,56.06,63.91
 Qwen2.5 32B,53.2,43.6,54.23,43.05,49.81,48.64
 Magistral Small 2506,41.02,23.48,41.25,20.36,40.27,51.89

results/results_icarus.json

@@ -1382,7 +1382,7 @@
         "Result": 92.95,
         "Model URL": "https://huggingface.co/openai/gpt-oss-120b",
         "Params": 120,
-        "Release": "V2",
+        "Release": "V3",
         "Thinking": "Reasoning"
     },
     {
@@ -1393,7 +1393,7 @@
         "Result": 83.83,
         "Model URL": "https://huggingface.co/openai/gpt-oss-120b",
         "Params": 120,
-        "Release": "V2",
+        "Release": "V3",
         "Thinking": "Reasoning"
     },
     {
@@ -1404,7 +1404,7 @@
         "Result": 78.08,
         "Model URL": "https://huggingface.co/openai/gpt-oss-120b",
         "Params": 120,
-        "Release": "V2",
+        "Release": "V3",
         "Thinking": "Reasoning"
     },
     {
@@ -1415,7 +1415,7 @@
         "Result": 60.0,
         "Model URL": "https://huggingface.co/openai/gpt-oss-120b",
         "Params": 120,
-        "Release": "V2",
+        "Release": "V3",
         "Thinking": "Reasoning"
     },
     {
@@ -1426,7 +1426,7 @@
         "Result": 77.56,
         "Model URL": "https://huggingface.co/openai/gpt-oss-120b",
         "Params": 120,
-        "Release": "V2",
+        "Release": "V3",
         "Thinking": "Reasoning"
     },
     {
@@ -1437,7 +1437,7 @@
         "Result": 51.49,
         "Model URL": "https://huggingface.co/openai/gpt-oss-120b",
         "Params": 120,
-        "Release": "V2",
+        "Release": "V3",
         "Thinking": "Reasoning"
     },
     {
@@ -1448,7 +1448,7 @@
         "Result": 76.66,
         "Model URL": "https://huggingface.co/openai/gpt-oss-120b",
         "Params": 120,
-        "Release": "V2",
+        "Release": "V3",
         "Thinking": "Reasoning"
     },
     {
@@ -1459,7 +1459,7 @@
         "Result": 53.96,
         "Model URL": "https://huggingface.co/openai/gpt-oss-120b",
         "Params": 120,
-        "Release": "V2",
+        "Release": "V3",
         "Thinking": "Reasoning"
     },
     {
@@ -1470,7 +1470,7 @@
         "Result": 74.49,
         "Model URL": "https://huggingface.co/openai/gpt-oss-120b",
         "Params": 120,
-        "Release": "V2",
+        "Release": "V3",
         "Thinking": "Reasoning"
     },
     {
@@ -1481,7 +1481,7 @@
         "Result": 49.85,
         "Model URL": "https://huggingface.co/openai/gpt-oss-120b",
         "Params": 120,
-        "Release": "V2",
+        "Release": "V3",
         "Thinking": "Reasoning"
     },
     {
@@ -1492,7 +1492,7 @@
         "Result": 76.53,
         "Model URL": "https://huggingface.co/openai/gpt-oss-120b",
         "Params": 120,
-        "Release": "V2",
+        "Release": "V3",
         "Thinking": "Reasoning"
     },
     {
@@ -1503,7 +1503,7 @@
         "Result": 54.64,
         "Model URL": "https://huggingface.co/openai/gpt-oss-120b",
         "Params": 120,
-        "Release": "V2",
+        "Release": "V3",
         "Thinking": "Reasoning"
     },
     {
@@ -1514,7 +1514,7 @@
         "Result": -1.0,
         "Model URL": "https://huggingface.co/openai/gpt-oss-120b",
         "Params": 120,
-        "Release": "V2",
+        "Release": "V3",
         "Thinking": "Reasoning"
     },
     {
@@ -1525,7 +1525,7 @@
         "Result": 95.64,
         "Model URL": "https://huggingface.co/openai/gpt-oss-120b",
         "Params": 120,
-        "Release": "V2",
+        "Release": "V3",
         "Thinking": "Reasoning"
     },
     {
@@ -1536,7 +1536,7 @@
         "Result": 88.24,
         "Model URL": "https://huggingface.co/openai/gpt-oss-120b",
         "Params": 120,
-        "Release": "V2",
+        "Release": "V3",
         "Thinking": "Reasoning"
     },
     {
@@ -1547,7 +1547,7 @@
         "Result": 81.15,
         "Model URL": "https://huggingface.co/openai/gpt-oss-120b",
         "Params": 120,
-        "Release": "V2",
+        "Release": "V3",
         "Thinking": "Reasoning"
     },
     {
@@ -1558,7 +1558,7 @@
         "Result": 68.24,
         "Model URL": "https://huggingface.co/openai/gpt-oss-120b",
         "Params": 120,
-        "Release": "V2",
+        "Release": "V3",
         "Thinking": "Reasoning"
     },
     {
@@ -1569,7 +1569,7 @@
         "Result": 80.51,
         "Model URL": "https://huggingface.co/openai/gpt-oss-120b",
         "Params": 120,
-        "Release": "V2",
+        "Release": "V3",
         "Thinking": "Reasoning"
     },
     {
@@ -1580,7 +1580,7 @@
         "Result": 68.24,
         "Model URL": "https://huggingface.co/openai/gpt-oss-120b",
         "Params": 120,
-        "Release": "V2",
+        "Release": "V3",
         "Thinking": "Reasoning"
     },
     {
@@ -1591,7 +1591,7 @@
         "Result": 79.72,
         "Model URL": "https://huggingface.co/openai/gpt-oss-120b",
         "Params": 120,
-        "Release": "V2",
+        "Release": "V3",
         "Thinking": "Reasoning"
     },
     {
@@ -1602,7 +1602,7 @@
         "Result": 70.1,
         "Model URL": "https://huggingface.co/openai/gpt-oss-120b",
         "Params": 120,
-        "Release": "V2",
+        "Release": "V3",
         "Thinking": "Reasoning"
     },
     {
@@ -1613,7 +1613,7 @@
         "Result": 77.54,
         "Model URL": "https://huggingface.co/openai/gpt-oss-120b",
         "Params": 120,
-        "Release": "V2",
+        "Release": "V3",
         "Thinking": "Reasoning"
     },
     {
@@ -1624,7 +1624,7 @@
         "Result": 68.27,
         "Model URL": "https://huggingface.co/openai/gpt-oss-120b",
         "Params": 120,
-        "Release": "V2",
+        "Release": "V3",
         "Thinking": "Reasoning"
     },
     {
@@ -1635,7 +1635,7 @@
         "Result": 78.97,
         "Model URL": "https://huggingface.co/openai/gpt-oss-120b",
         "Params": 120,
-        "Release": "V2",
+        "Release": "V3",
         "Thinking": "Reasoning"
     },
     {
@@ -1646,7 +1646,7 @@
         "Result": 69.85,
         "Model URL": "https://huggingface.co/openai/gpt-oss-120b",
         "Params": 120,
-        "Release": "V2",
+        "Release": "V3",
         "Thinking": "Reasoning"
     },
     {
@@ -2199,6 +2199,281 @@
         "Release": "V1",
         "Thinking": "Dense"
     },
+    {
+        "Model": "Seed-OSS-36B",
+        "Model Type": "General",
+        "Benchmark": "VerilogEval S2R",
+        "Task": "Syntax (STX)",
+        "Result": 88.97,
+        "Model URL": "https://huggingface.co/openai/gpt-oss-120b",
+        "Params": 36.2,
+        "Release": "V3",
+        "Thinking": "Reasoning"
+    },
+    {
+        "Model": "Seed-OSS-36B",
+        "Model Type": "General",
+        "Benchmark": "RTLLM",
+        "Task": "Syntax (STX)",
+        "Result": 81.7,
+        "Model URL": "https://huggingface.co/openai/gpt-oss-120b",
+        "Params": 36.2,
+        "Release": "V3",
+        "Thinking": "Reasoning"
+    },
+    {
+        "Model": "Seed-OSS-36B",
+        "Model Type": "General",
+        "Benchmark": "VerilogEval S2R",
+        "Task": "Functionality (FNC)",
+        "Result": 71.15,
+        "Model URL": "https://huggingface.co/openai/gpt-oss-120b",
+        "Params": 36.2,
+        "Release": "V3",
+        "Thinking": "Reasoning"
+    },
+    {
+        "Model": "Seed-OSS-36B",
+        "Model Type": "General",
+        "Benchmark": "RTLLM",
+        "Task": "Functionality (FNC)",
+        "Result": 59.15,
+        "Model URL": "https://huggingface.co/openai/gpt-oss-120b",
+        "Params": 36.2,
+        "Release": "V3",
+        "Thinking": "Reasoning"
+    },
+    {
+        "Model": "Seed-OSS-36B",
+        "Model Type": "General",
+        "Benchmark": "VerilogEval S2R",
+        "Task": "Synthesis (SYN)",
+        "Result": 71.15,
+        "Model URL": "https://huggingface.co/openai/gpt-oss-120b",
+        "Params": 36.2,
+        "Release": "V3",
+        "Thinking": "Reasoning"
+    },
+    {
+        "Model": "Seed-OSS-36B",
+        "Model Type": "General",
+        "Benchmark": "RTLLM",
+        "Task": "Synthesis (SYN)",
+        "Result": 56.6,
+        "Model URL": "https://huggingface.co/openai/gpt-oss-120b",
+        "Params": 36.2,
+        "Release": "V3",
+        "Thinking": "Reasoning"
+    },
+    {
+        "Model": "Seed-OSS-36B",
+        "Model Type": "General",
+        "Benchmark": "VerilogEval S2R",
+        "Task": "Power",
+        "Result": 70.1,
+        "Model URL": "https://huggingface.co/openai/gpt-oss-120b",
+        "Params": 36.2,
+        "Release": "V3",
+        "Thinking": "Reasoning"
+    },
+    {
+        "Model": "Seed-OSS-36B",
+        "Model Type": "General",
+        "Benchmark": "RTLLM",
+        "Task": "Power",
+        "Result": 64.63,
+        "Model URL": "https://huggingface.co/openai/gpt-oss-120b",
+        "Params": 36.2,
+        "Release": "V3",
+        "Thinking": "Reasoning"
+    },
+    {
+        "Model": "Seed-OSS-36B",
+        "Model Type": "General",
+        "Benchmark": "VerilogEval S2R",
+        "Task": "Performance",
+        "Result": 68.1,
+        "Model URL": "https://huggingface.co/openai/gpt-oss-120b",
+        "Params": 36.2,
+        "Release": "V3",
+        "Thinking": "Reasoning"
+    },
+    {
+        "Model": "Seed-OSS-36B",
+        "Model Type": "General",
+        "Benchmark": "RTLLM",
+        "Task": "Performance",
+        "Result": 57.32,
+        "Model URL": "https://huggingface.co/openai/gpt-oss-120b",
+        "Params": 36.2,
+        "Release": "V3",
+        "Thinking": "Reasoning"
+    },
+    {
+        "Model": "Seed-OSS-36B",
+        "Model Type": "General",
+        "Benchmark": "VerilogEval S2R",
+        "Task": "Area",
+        "Result": 70.32,
+        "Model URL": "https://huggingface.co/openai/gpt-oss-120b",
+        "Params": 36.2,
+        "Release": "V3",
+        "Thinking": "Reasoning"
+    },
+    {
+        "Model": "Seed-OSS-36B",
+        "Model Type": "General",
+        "Benchmark": "RTLLM",
+        "Task": "Area",
+        "Result": 60.88,
+        "Model URL": "https://huggingface.co/openai/gpt-oss-120b",
+        "Params": 36.2,
+        "Release": "V3",
+        "Thinking": "Reasoning"
+    },
+    {
+        "Model": "Seed-OSS-36B",
+        "Model Type": "General",
+        "Benchmark": "RTL-Repo",
+        "Task": "Exact Matching (EM)",
+        "Result": -1.0,
+        "Model URL": "https://huggingface.co/openai/gpt-oss-120b",
+        "Params": 36.2,
+        "Release": "V3",
+        "Thinking": "Reasoning"
+    },
+    {
+        "Model": "Seed-OSS-36B",
+        "Model Type": "General",
+        "Benchmark": "VerilogEval MC",
+        "Task": "Syntax (STX)",
+        "Result": 91.03,
+        "Model URL": "https://huggingface.co/openai/gpt-oss-120b",
+        "Params": 36.2,
+        "Release": "V3",
+        "Thinking": "Reasoning"
+    },
+    {
+        "Model": "Seed-OSS-36B",
+        "Model Type": "General",
+        "Benchmark": "VeriGen",
+        "Task": "Syntax (STX)",
+        "Result": 91.76,
+        "Model URL": "https://huggingface.co/openai/gpt-oss-120b",
+        "Params": 36.2,
+        "Release": "V3",
+        "Thinking": "Reasoning"
+    },
+    {
+        "Model": "Seed-OSS-36B",
+        "Model Type": "General",
+        "Benchmark": "VerilogEval MC",
+        "Task": "Functionality (FNC)",
+        "Result": 76.54,
+        "Model URL": "https://huggingface.co/openai/gpt-oss-120b",
+        "Params": 36.2,
+        "Release": "V3",
+        "Thinking": "Reasoning"
+    },
+    {
+        "Model": "Seed-OSS-36B",
+        "Model Type": "General",
+        "Benchmark": "VeriGen",
+        "Task": "Functionality (FNC)",
+        "Result": 70.59,
+        "Model URL": "https://huggingface.co/openai/gpt-oss-120b",
+        "Params": 36.2,
+        "Release": "V3",
+        "Thinking": "Reasoning"
+    },
+    {
+        "Model": "Seed-OSS-36B",
+        "Model Type": "General",
+        "Benchmark": "VerilogEval MC",
+        "Task": "Synthesis (SYN)",
+        "Result": 76.15,
+        "Model URL": "https://huggingface.co/openai/gpt-oss-120b",
+        "Params": 36.2,
+        "Release": "V3",
+        "Thinking": "Reasoning"
+    },
+    {
+        "Model": "Seed-OSS-36B",
+        "Model Type": "General",
+        "Benchmark": "VeriGen",
+        "Task": "Synthesis (SYN)",
+        "Result": 70.59,
+        "Model URL": "https://huggingface.co/openai/gpt-oss-120b",
+        "Params": 36.2,
+        "Release": "V3",
+        "Thinking": "Reasoning"
+    },
+    {
+        "Model": "Seed-OSS-36B",
+        "Model Type": "General",
+        "Benchmark": "VerilogEval MC",
+        "Task": "Power",
+        "Result": 74.95,
+        "Model URL": "https://huggingface.co/openai/gpt-oss-120b",
+        "Params": 36.2,
+        "Release": "V3",
+        "Thinking": "Reasoning"
+    },
+    {
+        "Model": "Seed-OSS-36B",
+        "Model Type": "General",
+        "Benchmark": "VeriGen",
+        "Task": "Power",
+        "Result": 66.11,
+        "Model URL": "https://huggingface.co/openai/gpt-oss-120b",
+        "Params": 36.2,
+        "Release": "V3",
+        "Thinking": "Reasoning"
+    },
+    {
+        "Model": "Seed-OSS-36B",
+        "Model Type": "General",
+        "Benchmark": "VerilogEval MC",
+        "Task": "Performance",
+        "Result": 73.33,
+        "Model URL": "https://huggingface.co/openai/gpt-oss-120b",
+        "Params": 36.2,
+        "Release": "V3",
+        "Thinking": "Reasoning"
+    },
+    {
+        "Model": "Seed-OSS-36B",
+        "Model Type": "General",
+        "Benchmark": "VeriGen",
+        "Task": "Performance",
+        "Result": 70.19,
+        "Model URL": "https://huggingface.co/openai/gpt-oss-120b",
+        "Params": 36.2,
+        "Release": "V3",
+        "Thinking": "Reasoning"
+    },
+    {
+        "Model": "Seed-OSS-36B",
+        "Model Type": "General",
+        "Benchmark": "VerilogEval MC",
+        "Task": "Area",
+        "Result": 74.69,
+        "Model URL": "https://huggingface.co/openai/gpt-oss-120b",
+        "Params": 36.2,
+        "Release": "V3",
+        "Thinking": "Reasoning"
+    },
+    {
+        "Model": "Seed-OSS-36B",
+        "Model Type": "General",
+        "Benchmark": "VeriGen",
+        "Task": "Area",
+        "Result": 72.45,
+        "Model URL": "https://huggingface.co/openai/gpt-oss-120b",
+        "Params": 36.2,
+        "Release": "V3",
+        "Thinking": "Reasoning"
+    },
     {
         "Model": "QwQ 32B",
         "Model Type": "General",
@@ -3032,7 +3307,7 @@
         "Result": 88.08,
         "Model URL": "https://huggingface.co/openai/gpt-oss-20b",
         "Params": 21.5,
-        "Release": "V2",
+        "Release": "V3",
         "Thinking": "Reasoning"
     },
     {
@@ -3043,7 +3318,7 @@
         "Result": 86.81,
         "Model URL": "https://huggingface.co/openai/gpt-oss-20b",
         "Params": 21.5,
-        "Release": "V2",
+        "Release": "V3",
         "Thinking": "Reasoning"
     },
     {
@@ -3054,7 +3329,7 @@
         "Result": 69.74,
         "Model URL": "https://huggingface.co/openai/gpt-oss-20b",
         "Params": 21.5,
-        "Release": "V2",
+        "Release": "V3",
         "Thinking": "Reasoning"
     },
     {
@@ -3065,7 +3340,7 @@
         "Result": 58.72,
         "Model URL": "https://huggingface.co/openai/gpt-oss-20b",
         "Params": 21.5,
-        "Release": "V2",
+        "Release": "V3",
         "Thinking": "Reasoning"
     },
     {
@@ -3076,7 +3351,7 @@
         "Result": 69.23,
         "Model URL": "https://huggingface.co/openai/gpt-oss-20b",
         "Params": 21.5,
-        "Release": "V2",
+        "Release": "V3",
         "Thinking": "Reasoning"
     },
     {
@@ -3087,7 +3362,7 @@
         "Result": 48.94,
         "Model URL": "https://huggingface.co/openai/gpt-oss-20b",
         "Params": 21.5,
-        "Release": "V2",
+        "Release": "V3",
         "Thinking": "Reasoning"
     },
     {
@@ -3098,7 +3373,7 @@
         "Result": 69.85,
         "Model URL": "https://huggingface.co/openai/gpt-oss-20b",
         "Params": 21.5,
-        "Release": "V2",
+        "Release": "V3",
         "Thinking": "Reasoning"
     },
     {
@@ -3109,7 +3384,7 @@
         "Result": 55.89,
         "Model URL": "https://huggingface.co/openai/gpt-oss-20b",
         "Params": 21.5,
-        "Release": "V2",
+        "Release": "V3",
         "Thinking": "Reasoning"
     },
     {
@@ -3120,7 +3395,7 @@
         "Result": 64.57,
         "Model URL": "https://huggingface.co/openai/gpt-oss-20b",
         "Params": 21.5,
-        "Release": "V2",
+        "Release": "V3",
         "Thinking": "Reasoning"
     },
     {
@@ -3131,7 +3406,7 @@
         "Result": 45.21,
         "Model URL": "https://huggingface.co/openai/gpt-oss-20b",
         "Params": 21.5,
-        "Release": "V2",
+        "Release": "V3",
         "Thinking": "Reasoning"
     },
     {
@@ -3142,7 +3417,7 @@
         "Result": 67.82,
         "Model URL": "https://huggingface.co/openai/gpt-oss-20b",
         "Params": 21.5,
-        "Release": "V2",
+        "Release": "V3",
         "Thinking": "Reasoning"
     },
     {
@@ -3153,7 +3428,7 @@
         "Result": 53.29,
         "Model URL": "https://huggingface.co/openai/gpt-oss-20b",
         "Params": 21.5,
-        "Release": "V2",
+        "Release": "V3",
         "Thinking": "Reasoning"
     },
     {
@@ -3164,7 +3439,7 @@
         "Result": -1.0,
         "Model URL": "https://huggingface.co/openai/gpt-oss-20b",
         "Params": 21.5,
-        "Release": "V2",
+        "Release": "V3",
         "Thinking": "Reasoning"
     },
     {
@@ -3175,7 +3450,7 @@
         "Result": 91.92,
         "Model URL": "https://huggingface.co/openai/gpt-oss-20b",
         "Params": 21.5,
-        "Release": "V2",
+        "Release": "V3",
         "Thinking": "Reasoning"
     },
     {
@@ -3186,7 +3461,7 @@
         "Result": 76.47,
         "Model URL": "https://huggingface.co/openai/gpt-oss-20b",
         "Params": 21.5,
-        "Release": "V2",
+        "Release": "V3",
         "Thinking": "Reasoning"
     },
     {
@@ -3197,7 +3472,7 @@
         "Result": 70.26,
         "Model URL": "https://huggingface.co/openai/gpt-oss-20b",
         "Params": 21.5,
-        "Release": "V2",
+        "Release": "V3",
         "Thinking": "Reasoning"
     },
     {
@@ -3208,7 +3483,7 @@
         "Result": 50.59,
         "Model URL": "https://huggingface.co/openai/gpt-oss-20b",
         "Params": 21.5,
-        "Release": "V2",
+        "Release": "V3",
         "Thinking": "Reasoning"
     },
     {
@@ -3219,7 +3494,7 @@
         "Result": 69.74,
         "Model URL": "https://huggingface.co/openai/gpt-oss-20b",
         "Params": 21.5,
-        "Release": "V2",
+        "Release": "V3",
         "Thinking": "Reasoning"
     },
     {
@@ -3230,7 +3505,7 @@
         "Result": 50.59,
         "Model URL": "https://huggingface.co/openai/gpt-oss-20b",
         "Params": 21.5,
-        "Release": "V2",
+        "Release": "V3",
         "Thinking": "Reasoning"
     },
     {
@@ -3241,7 +3516,7 @@
         "Result": 69.5,
         "Model URL": "https://huggingface.co/openai/gpt-oss-20b",
         "Params": 21.5,
-        "Release": "V2",
+        "Release": "V3",
         "Thinking": "Reasoning"
     },
     {
@@ -3252,7 +3527,7 @@
         "Result": 45.41,
         "Model URL": "https://huggingface.co/openai/gpt-oss-20b",
         "Params": 21.5,
-        "Release": "V2",
+        "Release": "V3",
         "Thinking": "Reasoning"
     },
     {
@@ -3263,7 +3538,7 @@
         "Result": 67.06,
         "Model URL": "https://huggingface.co/openai/gpt-oss-20b",
         "Params": 21.5,
-        "Release": "V2",
+        "Release": "V3",
         "Thinking": "Reasoning"
     },
     {
@@ -3274,7 +3549,7 @@
         "Result": 50.48,
         "Model URL": "https://huggingface.co/openai/gpt-oss-20b",
         "Params": 21.5,
-        "Release": "V2",
+        "Release": "V3",
         "Thinking": "Reasoning"
     },
     {
@@ -3285,7 +3560,7 @@
         "Result": 68.76,
         "Model URL": "https://huggingface.co/openai/gpt-oss-20b",
         "Params": 21.5,
-        "Release": "V2",
+        "Release": "V3",
         "Thinking": "Reasoning"
     },
     {
@@ -3296,7 +3571,7 @@
         "Result": 49.85,
         "Model URL": "https://huggingface.co/openai/gpt-oss-20b",
         "Params": 21.5,
-        "Release": "V2",
+        "Release": "V3",
         "Thinking": "Reasoning"
     },
     {

results/results_icarus_october_2025.csv

@@ -0,0 +1,67 @@
+,Syntax (STX),Syntax (STX),Functionality (FNC),Functionality (FNC),Synthesis (SYN),Synthesis (SYN),Power,Power,Performance,Performance,Area,Area,EM,Syntax (STX),Syntax (STX),Functionality (FNC),Functionality (FNC),Synthesis (SYN),Synthesis (SYN),Power,Power,Performance,Performance,Area,Area
+,VerilogEval S2R,RTLLM,VerilogEval S2R,RTLLM,VerilogEval S2R,RTLLM,VerilogEval S2R,RTLLM,VerilogEval S2R,RTLLM,VerilogEval S2R,RTLLM,RTL-Repo,VerilogEval MC,VeriGen,VerilogEval MC,VeriGen,VerilogEval MC,VeriGen,VerilogEval MC,VeriGen,VerilogEval MC,VeriGen,VerilogEval MC,VeriGen
+DeepSeek R1-0528,96.150000000000006,86.120000000000005,81.540000000000006,64.900000000000006,81.280000000000001,64.489999999999995,79.150000000000006,73.329999999999998,78.090000000000003,65.640000000000001,79.280000000000001,71.150000000000006,-1,95.379999999999995,91.760000000000005,81.540000000000006,74.120000000000005,81.409999999999997,74.120000000000005,80.269999999999996,63.859999999999999,78.689999999999998,74.049999999999997,79.989999999999995,77.010000000000005
+DeepSeek R1,97.180000000000007,89.799999999999997,79.739999999999995,65.709999999999994,79.620000000000005,63.270000000000003,78.329999999999998,71.340000000000003,76.489999999999995,64.060000000000002,78.189999999999998,70.079999999999998,-1,97.439999999999998,96.469999999999999,79.489999999999995,74.120000000000005,79.489999999999995,74.120000000000005,78.269999999999996,64.340000000000003,76.430000000000007,74.290000000000006,77.959999999999994,77.120000000000005
+Qwen3 Coder 480B A35B,96.409999999999997,86.379999999999995,65.379999999999995,59.57,64.739999999999995,53.619999999999997,64.090000000000003,52.850000000000001,61.909999999999997,53.030000000000001,62.93,51.810000000000002,38.880000000000003,84.099999999999994,100,58.850000000000001,75.290000000000006,58.460000000000001,75.290000000000006,57.049999999999997,69.629999999999995,55.539999999999999,73.950000000000003,55.789999999999999,76.760000000000005
+Llama 3.1 405B,87.439999999999998,77.140000000000001,58.969999999999999,45.710000000000001,58.850000000000001,41.630000000000003,57.579999999999998,50.880000000000003,55.93,32.439999999999998,56.130000000000003,43.450000000000003,34.619999999999997,88.590000000000003,95.290000000000006,56.149999999999999,58.82,55.899999999999999,58.82,55.130000000000003,55.100000000000001,53.450000000000003,58.399999999999999,54.479999999999997,61.200000000000003
+Qwen3 236B A22B,91.280000000000001,73.879999999999995,76.920000000000002,51.43,76.790000000000006,48.57,75.25,54.609999999999999,73.560000000000002,46.369999999999997,75.670000000000002,50.469999999999999,41.939999999999998,82.180000000000007,87.060000000000002,69.620000000000005,62.350000000000001,69.620000000000005,62.350000000000001,69.040000000000006,54.630000000000003,66.890000000000001,62.630000000000003,69.150000000000006,62.780000000000001
+gpt-oss-120b,92.950000000000003,83.829999999999998,78.079999999999998,60,77.560000000000002,51.490000000000002,76.659999999999997,53.960000000000001,74.489999999999995,49.850000000000001,76.530000000000001,54.640000000000001,-1,95.640000000000001,88.239999999999995,81.150000000000006,68.239999999999995,80.510000000000005,68.239999999999995,79.719999999999999,70.099999999999994,77.540000000000006,68.269999999999996,78.969999999999999,69.849999999999994
+Qwen2.5 72B,82.180000000000007,79.590000000000003,52.439999999999998,45.310000000000002,51.920000000000002,44.079999999999998,51.829999999999998,46.469999999999999,48.75,45.399999999999999,50.090000000000003,47.649999999999999,37.439999999999998,80.900000000000006,84.709999999999994,52.950000000000003,47.060000000000002,52.689999999999998,47.060000000000002,51.659999999999997,47.590000000000003,49.369999999999997,46.960000000000001,51.18,47.700000000000003
+Llama 3.(1-3) 70B,66.150000000000006,73.879999999999995,40.640000000000001,42.450000000000003,40.640000000000001,39.18,40.460000000000001,40.810000000000002,38.079999999999998,38.140000000000001,39.859999999999999,39.649999999999999,28.719999999999999,84.739999999999995,89.409999999999997,41.670000000000002,65.879999999999995,41.670000000000002,64.709999999999994,41.380000000000003,63.469999999999999,39.75,64.689999999999998,41.359999999999999,64.709999999999994
+Seed-OSS-36B,88.969999999999999,81.700000000000003,71.150000000000006,59.149999999999999,71.150000000000006,56.600000000000001,70.099999999999994,64.629999999999995,68.099999999999994,57.32,70.319999999999993,60.880000000000003,-1,91.030000000000001,91.760000000000005,76.540000000000006,70.590000000000003,76.150000000000006,70.590000000000003,74.950000000000003,66.109999999999999,73.329999999999998,70.189999999999998,74.689999999999998,72.450000000000003
+QwQ 32B,87.950000000000003,82.450000000000003,66.409999999999997,56.729999999999997,66.409999999999997,52.240000000000002,66.150000000000006,55.829999999999998,63.799999999999997,51.909999999999997,65.120000000000005,56.07,-1,58.969999999999999,68.239999999999995,40,52.939999999999998,39.619999999999997,52.939999999999998,39.399999999999999,51.469999999999999,37.530000000000001,52.93,39.100000000000001,53.390000000000001
+Qwen2.5 32B,88.590000000000003,84.079999999999998,52.560000000000002,50.200000000000003,52.18,46.119999999999997,52.32,49.729999999999997,49.43,46.43,50.82,50.43,28.93,93.209999999999994,85.879999999999995,41.539999999999999,45.880000000000003,41.539999999999999,45.880000000000003,41.310000000000002,43.560000000000002,40.479999999999997,46.079999999999998,41.229999999999997,45.369999999999997
+Magistral Small 2506,63.850000000000001,71.909999999999997,42.950000000000003,43.399999999999999,42.439999999999998,37.450000000000003,41.390000000000001,43.359999999999999,40.090000000000003,37.439999999999998,41.390000000000001,40.32,-1,31.280000000000001,68.239999999999995,20.899999999999999,50.590000000000003,20.77,49.409999999999997,20.109999999999999,41.590000000000003,19.609999999999999,48.969999999999999,20.16,50.07
+gpt-oss-20b,88.079999999999998,86.810000000000002,69.739999999999995,58.719999999999999,69.230000000000004,48.939999999999998,69.849999999999994,55.890000000000001,64.569999999999993,45.210000000000001,67.819999999999993,53.289999999999999,-1,91.920000000000002,76.469999999999999,70.260000000000005,50.590000000000003,69.739999999999995,50.590000000000003,69.5,45.409999999999997,67.060000000000002,50.479999999999997,68.760000000000005,49.850000000000001
+StarChat2 15B v0.1,88.459999999999994,84.900000000000006,37.950000000000003,44.490000000000002,37.950000000000003,44.079999999999998,37.560000000000002,46.950000000000003,35.299999999999997,43.219999999999999,37.189999999999998,46.649999999999999,13.42,79.739999999999995,92.939999999999998,36.409999999999997,63.530000000000001,36.030000000000001,63.530000000000001,36.079999999999998,58.060000000000002,34.909999999999997,63.259999999999998,35.759999999999998,64.560000000000002
+DeepSeek R1 Distill Qwen 14B,42.18,34.689999999999998,25.510000000000002,18.370000000000001,25.510000000000002,16.329999999999998,25.359999999999999,17.859999999999999,24.190000000000001,16.48,25.27,17.329999999999998,-1,45,44.710000000000001,25.640000000000001,28.239999999999998,25.260000000000002,28.239999999999998,24.789999999999999,24.710000000000001,23.48,28.140000000000001,24.629999999999999,28.350000000000001
+CodeLlama 70B,67.049999999999997,69.799999999999997,33.079999999999998,36.329999999999998,33.079999999999998,34.289999999999999,32.689999999999998,37.189999999999998,31.460000000000001,34.289999999999999,32.439999999999998,35.950000000000003,24.329999999999998,90.769999999999996,88.239999999999995,33.329999999999998,47.060000000000002,33.329999999999998,47.060000000000002,33.020000000000003,45.799999999999997,30.800000000000001,46.909999999999997,32.990000000000002,46.979999999999997
+DeepSeek Coder 33B,62.82,83.670000000000002,23.329999999999998,42.450000000000003,23.079999999999998,42.039999999999999,22.859999999999999,42.289999999999999,22.809999999999999,39.420000000000002,22.289999999999999,42.710000000000001,24.579999999999998,75.260000000000005,88.239999999999995,39.619999999999997,45.880000000000003,39.359999999999999,45.880000000000003,38.229999999999997,46.259999999999998,36.789999999999999,45.609999999999999,37.899999999999999,46.170000000000002
+QwenCoder 2.5 32B,87.180000000000007,77.959999999999994,45,43.270000000000003,44.869999999999997,43.270000000000003,44.25,46.82,43.030000000000001,43.200000000000003,43.759999999999998,45.420000000000002,31.07,83.719999999999999,87.060000000000002,45.640000000000001,54.119999999999997,45.130000000000003,54.119999999999997,44.590000000000003,54.549999999999997,43.009999999999998,54.009999999999998,44.549999999999997,55.009999999999998
+DeepCoder 14B,43.850000000000001,39.590000000000003,28.079999999999998,23.670000000000002,28.079999999999998,22.039999999999999,27.940000000000001,25,26.260000000000002,22,27.77,23.149999999999999,-1,61.920000000000002,48.240000000000002,34.100000000000001,32.939999999999998,33.719999999999999,32.939999999999998,33.700000000000003,30.469999999999999,32.170000000000002,32.840000000000003,33.670000000000002,32.909999999999997
+QwenCoder 2.5 14B,78.969999999999999,81.629999999999995,37.82,46.119999999999997,37.439999999999998,45.310000000000002,35.939999999999998,45.82,34.829999999999998,44.640000000000001,35.18,46.049999999999997,37.530000000000001,80,83.530000000000001,41.670000000000002,48.240000000000002,41.149999999999999,48.240000000000002,40.740000000000002,47.090000000000003,39.200000000000003,48.289999999999999,40.829999999999998,47.539999999999999
+SeedCoder 8B,91.409999999999997,85.310000000000002,53.460000000000001,47.350000000000001,53.329999999999998,46.530000000000001,52.859999999999999,49.420000000000002,50.619999999999997,45.600000000000001,51.649999999999999,49.590000000000003,28.23,77.439999999999998,94.120000000000005,37.310000000000002,42.350000000000001,37.310000000000002,38.82,37.32,35.289999999999999,35.350000000000001,38.689999999999998,36.890000000000001,38.990000000000002
+SeedCoder 8B Reasoning,67.819999999999993,53.469999999999999,49.229999999999997,30.199999999999999,49.229999999999997,29.390000000000001,48.920000000000002,32.039999999999999,46.759999999999998,28.640000000000001,47.869999999999997,29.989999999999998,-1,83.329999999999998,78.819999999999993,48.210000000000001,62.350000000000001,48.079999999999998,62.350000000000001,47.780000000000001,53.479999999999997,45.439999999999998,61.810000000000002,47.060000000000002,64.620000000000005
+OpenCoder 8B,78.209999999999994,75.920000000000002,28.460000000000001,42.859999999999999,27.82,40.82,27.34,41.359999999999999,25.949999999999999,39.770000000000003,27.109999999999999,41.359999999999999,16.170000000000002,80,95.290000000000006,35.640000000000001,45.880000000000003,35.380000000000003,45.880000000000003,35.119999999999997,42.399999999999999,33.469999999999999,45.75,35.130000000000003,46.259999999999998
+QwenCoder 2.5 7B,20.129999999999999,76.329999999999998,6.9199999999999999,38.780000000000001,6.6699999999999999,37.140000000000001,6.5099999999999998,40.649999999999999,6.6299999999999999,37.25,6.5599999999999996,39.579999999999998,28.329999999999998,74.099999999999994,90.590000000000003,33.719999999999999,40,33.719999999999999,40,33.590000000000003,37.729999999999997,31.780000000000001,40.07,33.619999999999997,40.57
+DeepSeek Coder 6.7B,82.049999999999997,78.780000000000001,29.620000000000001,41.219999999999999,29.489999999999998,38.780000000000001,29.510000000000002,42.619999999999997,27.73,39.329999999999998,29.41,43.299999999999997,24.629999999999999,67.180000000000007,84.709999999999994,31.670000000000002,37.649999999999999,29.870000000000001,37.649999999999999,29.780000000000001,36.450000000000003,27.98,37.469999999999999,29.210000000000001,37.659999999999997
+CodeV R1 Distill Qwen 7B,56.920000000000002,73.060000000000002,33.329999999999998,49.799999999999997,33.329999999999998,47.350000000000001,32.579999999999998,49.25,32.009999999999998,47.450000000000003,32.450000000000003,49.009999999999998,-1,92.689999999999998,89.409999999999997,21.280000000000001,65.879999999999995,21.280000000000001,65.879999999999995,21.039999999999999,60.100000000000001,19.59,65.590000000000003,21.050000000000001,66.239999999999995
+HaVen-CodeQwen,93.329999999999998,80.409999999999997,47.310000000000002,42.859999999999999,46.149999999999999,41.219999999999999,45.079999999999998,40.590000000000003,44.259999999999998,38.829999999999998,44.68,40.530000000000001,25.140000000000001,93.590000000000003,100,50.130000000000003,62.350000000000001,49.490000000000002,62.350000000000001,47.549999999999997,61.82,47.049999999999997,62.530000000000001,47.090000000000003,61.759999999999998
+CodeV-QW-7B,45.380000000000003,68.159999999999997,19.620000000000001,34.289999999999999,18.969999999999999,26.530000000000001,18.91,28.140000000000001,18.710000000000001,21.800000000000001,18.850000000000001,26.5,20.940000000000001,93.329999999999998,100,52.310000000000002,60,51.539999999999999,60,51.689999999999998,59.899999999999999,48.789999999999999,59.950000000000003,51.450000000000003,60.579999999999998
+RTLCoder Mistral,54.869999999999997,32.240000000000002,24.620000000000001,16.329999999999998,24.620000000000001,15.92,24.280000000000001,16.030000000000001,22.780000000000001,14.710000000000001,24.059999999999999,16,14.77,60.509999999999998,85.879999999999995,27.050000000000001,51.759999999999998,27.050000000000001,51.759999999999998,26.940000000000001,49.850000000000001,25.219999999999999,51.939999999999998,26.870000000000001,52.719999999999999
+RTLCoder DeepSeek,84.620000000000005,73.060000000000002,39.490000000000002,37.140000000000001,39.490000000000002,34.689999999999998,38.909999999999997,34.299999999999997,37.520000000000003,32.759999999999998,38.549999999999997,33.689999999999998,19.350000000000001,77.310000000000002,85.879999999999995,36.920000000000002,55.289999999999999,36.789999999999999,55.289999999999999,36.939999999999998,50.789999999999999,34.840000000000003,55.219999999999999,36.619999999999997,54.689999999999998
+OriGen,96.150000000000006,81.629999999999995,54.229999999999997,50.609999999999999,54.229999999999997,50.609999999999999,54.289999999999999,53.100000000000001,51.57,50.859999999999999,53.149999999999999,53.439999999999998,17.07,92.439999999999998,98.819999999999993,50.770000000000003,76.469999999999999,50.770000000000003,76.469999999999999,50.950000000000003,71.659999999999997,48.530000000000001,76.609999999999999,50.509999999999998,78.700000000000003
+CodeV-CL-7B,32.18,48.159999999999997,13.08,24.489999999999998,12.949999999999999,21.629999999999999,12.800000000000001,22.25,12.51,20.59,12.82,21.289999999999999,12.27,92.049999999999997,98.819999999999993,31.789999999999999,49.409999999999997,31.789999999999999,49.409999999999997,31.739999999999998,48.130000000000003,29.449999999999999,49.340000000000003,31.609999999999999,49.079999999999998
+CodeV-DS-6.7B,33.590000000000003,67.349999999999994,15,38.780000000000001,15,37.140000000000001,15.1,35.560000000000002,14.460000000000001,35.130000000000003,14.85,35.880000000000003,21.260000000000002,95.510000000000005,100,47.049999999999997,61.18,47.049999999999997,60,47.369999999999997,59.880000000000003,44.350000000000001,59.960000000000001,46.520000000000003,59.770000000000003
+,,,,,,,,,,,,,,,,,,,,,,,,,
+,,,,,,,,,,,,,,,,,,,,,,,,,
+,,,,,,,,,,,,,,,,,,,,,,,,,
+,,,,,,,,,,,,,,,,,,,,,,,,,
+,,,,,,,,,,,,,,,,,,,,,,,,,
+,,,,,,,,,,,,,,,,,,,,,,,,,
+,,,,,,,,,,,,,,,,,,,,,,,,,
+,,,,,,,,,,,,,,,,,,,,,,,,,
+,,,,,,,,,,,,,,,,,,,,,,,,,
+,,,,,,,,,,,,,,,,,,,,,,,,,
+,,,,,,,,,,,,,,,,,,,,,,,,,
+,,,,,,,,,,,,,,,,,,,,,,,,,
+,,,,,,,,,,,,,,,,,,,,,,,,,
+,,,,,,,,,,,,,,,,,,,,,,,,,
+,,,,,,,,,,,,,,,,,,,,,,,,,
+,,,,,,,,,,,,,,,,,,,,,,,,,
+,,,,,,,,,,,,,,,,,,,,,,,,,
+,,,,,,,,,,,,,,,,,,,,,,,,,
+,,,,,,,,,,,,,,,,,,,,,,,,,
+,,,,,,,,,,,,,,,,,,,,,,,,,
+,,,,,,,,,,,,,,,,,,,,,,,,,
+,,,,,,,,,,,,,,,,,,,,,,,,,
+,,,,,,,,,,,,,,,,,,,,,,,,,
+,,,,,,,,,,,,,,,,,,,,,,,,,
+,,,,,,,,,,,,,,,,,,,,,,,,,
+,,,,,,,,,,,,,,,,,,,,,,,,,
+,,,,,,,,,,,,,,,,,,,,,,,,,
+,,,,,,,,,,,,,,,,,,,,,,,,,
+,,,,,,,,,,,,,,,,,,,,,,,,,
+,,,,,,,,,,,,,,,,,,,,,,,,,
+,,,,,,,,,,,,,,,,,,,,,,,,,
+

results/results_verilator.json

@@ -1382,7 +1382,7 @@
         "Result": 94.36,
         "Model URL": "https://huggingface.co/openai/gpt-oss-120b",
         "Params": 120,
-        "Release": "V2",
+        "Release": "V3",
         "Thinking": "Reasoning"
     },
     {
@@ -1393,7 +1393,7 @@
         "Result": 85.96,
         "Model URL": "https://huggingface.co/openai/gpt-oss-120b",
         "Params": 120,
-        "Release": "V2",
+        "Release": "V3",
         "Thinking": "Reasoning"
     },
     {
@@ -1404,7 +1404,7 @@
         "Result": 76.67,
         "Model URL": "https://huggingface.co/openai/gpt-oss-120b",
         "Params": 120,
-        "Release": "V2",
+        "Release": "V3",
         "Thinking": "Reasoning"
     },
     {
@@ -1415,7 +1415,7 @@
         "Result": 62.13,
         "Model URL": "https://huggingface.co/openai/gpt-oss-120b",
         "Params": 120,
-        "Release": "V2",
+        "Release": "V3",
         "Thinking": "Reasoning"
     },
     {
@@ -1426,7 +1426,7 @@
         "Result": 76.54,
         "Model URL": "https://huggingface.co/openai/gpt-oss-120b",
         "Params": 120,
-        "Release": "V2",
+        "Release": "V3",
         "Thinking": "Reasoning"
     },
     {
@@ -1437,7 +1437,7 @@
         "Result": 51.49,
         "Model URL": "https://huggingface.co/openai/gpt-oss-120b",
         "Params": 120,
-        "Release": "V2",
+        "Release": "V3",
         "Thinking": "Reasoning"
     },
     {
@@ -1448,7 +1448,7 @@
         "Result": 76.51,
         "Model URL": "https://huggingface.co/openai/gpt-oss-120b",
         "Params": 120,
-        "Release": "V2",
+        "Release": "V3",
         "Thinking": "Reasoning"
     },
     {
@@ -1459,7 +1459,7 @@
         "Result": 53.96,
         "Model URL": "https://huggingface.co/openai/gpt-oss-120b",
         "Params": 120,
-        "Release": "V2",
+        "Release": "V3",
         "Thinking": "Reasoning"
     },
     {
@@ -1470,7 +1470,7 @@
         "Result": 73.83,
         "Model URL": "https://huggingface.co/openai/gpt-oss-120b",
         "Params": 120,
-        "Release": "V2",
+        "Release": "V3",
         "Thinking": "Reasoning"
     },
     {
@@ -1481,7 +1481,7 @@
         "Result": 49.85,
         "Model URL": "https://huggingface.co/openai/gpt-oss-120b",
         "Params": 120,
-        "Release": "V2",
+        "Release": "V3",
         "Thinking": "Reasoning"
     },
     {
@@ -1492,7 +1492,7 @@
         "Result": 76.01,
         "Model URL": "https://huggingface.co/openai/gpt-oss-120b",
         "Params": 120,
-        "Release": "V2",
+        "Release": "V3",
         "Thinking": "Reasoning"
     },
     {
@@ -1503,7 +1503,7 @@
         "Result": 54.64,
         "Model URL": "https://huggingface.co/openai/gpt-oss-120b",
         "Params": 120,
-        "Release": "V2",
+        "Release": "V3",
         "Thinking": "Reasoning"
     },
     {
@@ -1514,7 +1514,7 @@
         "Result": -1.0,
         "Model URL": "https://huggingface.co/openai/gpt-oss-120b",
         "Params": 120,
-        "Release": "V2",
+        "Release": "V3",
         "Thinking": "Reasoning"
     },
     {
@@ -1525,7 +1525,7 @@
         "Result": 95.38,
         "Model URL": "https://huggingface.co/openai/gpt-oss-120b",
         "Params": 120,
-        "Release": "V2",
+        "Release": "V3",
         "Thinking": "Reasoning"
     },
     {
@@ -1536,7 +1536,7 @@
         "Result": 89.41,
         "Model URL": "https://huggingface.co/openai/gpt-oss-120b",
         "Params": 120,
-        "Release": "V2",
+        "Release": "V3",
         "Thinking": "Reasoning"
     },
     {
@@ -1547,7 +1547,7 @@
         "Result": 76.92,
         "Model URL": "https://huggingface.co/openai/gpt-oss-120b",
         "Params": 120,
-        "Release": "V2",
+        "Release": "V3",
         "Thinking": "Reasoning"
     },
     {
@@ -1558,7 +1558,7 @@
         "Result": 69.41,
         "Model URL": "https://huggingface.co/openai/gpt-oss-120b",
         "Params": 120,
-        "Release": "V2",
+        "Release": "V3",
         "Thinking": "Reasoning"
     },
     {
@@ -1569,7 +1569,7 @@
         "Result": 76.67,
         "Model URL": "https://huggingface.co/openai/gpt-oss-120b",
         "Params": 120,
-        "Release": "V2",
+        "Release": "V3",
         "Thinking": "Reasoning"
     },
     {
@@ -1580,7 +1580,7 @@
         "Result": 69.41,
         "Model URL": "https://huggingface.co/openai/gpt-oss-120b",
         "Params": 120,
-        "Release": "V2",
+        "Release": "V3",
         "Thinking": "Reasoning"
     },
     {
@@ -1591,7 +1591,7 @@
         "Result": 76.48,
         "Model URL": "https://huggingface.co/openai/gpt-oss-120b",
         "Params": 120,
-        "Release": "V2",
+        "Release": "V3",
         "Thinking": "Reasoning"
     },
     {
@@ -1602,7 +1602,7 @@
         "Result": 71.05,
         "Model URL": "https://huggingface.co/openai/gpt-oss-120b",
         "Params": 120,
-        "Release": "V2",
+        "Release": "V3",
         "Thinking": "Reasoning"
     },
     {
@@ -1613,7 +1613,7 @@
         "Result": 73.91,
         "Model URL": "https://huggingface.co/openai/gpt-oss-120b",
         "Params": 120,
-        "Release": "V2",
+        "Release": "V3",
         "Thinking": "Reasoning"
     },
     {
@@ -1624,7 +1624,7 @@
         "Result": 69.44,
         "Model URL": "https://huggingface.co/openai/gpt-oss-120b",
         "Params": 120,
-        "Release": "V2",
+        "Release": "V3",
         "Thinking": "Reasoning"
     },
     {
@@ -1635,7 +1635,7 @@
         "Result": 75.81,
         "Model URL": "https://huggingface.co/openai/gpt-oss-120b",
         "Params": 120,
-        "Release": "V2",
+        "Release": "V3",
         "Thinking": "Reasoning"
     },
     {
@@ -1646,7 +1646,7 @@
         "Result": 70.88,
         "Model URL": "https://huggingface.co/openai/gpt-oss-120b",
         "Params": 120,
-        "Release": "V2",
+        "Release": "V3",
         "Thinking": "Reasoning"
     },
     {
@@ -2199,6 +2199,281 @@
         "Release": "V1",
         "Thinking": "Dense"
     },
+    {
+        "Model": "Seed-OSS-36B",
+        "Model Type": "General",
+        "Benchmark": "VerilogEval S2R",
+        "Task": "Syntax (STX)",
+        "Result": 89.36,
+        "Model URL": "https://huggingface.co/openai/gpt-oss-120b",
+        "Params": 36.2,
+        "Release": "V3",
+        "Thinking": "Reasoning"
+    },
+    {
+        "Model": "Seed-OSS-36B",
+        "Model Type": "General",
+        "Benchmark": "RTLLM",
+        "Task": "Syntax (STX)",
+        "Result": 83.4,
+        "Model URL": "https://huggingface.co/openai/gpt-oss-120b",
+        "Params": 36.2,
+        "Release": "V3",
+        "Thinking": "Reasoning"
+    },
+    {
+        "Model": "Seed-OSS-36B",
+        "Model Type": "General",
+        "Benchmark": "VerilogEval S2R",
+        "Task": "Functionality (FNC)",
+        "Result": 71.15,
+        "Model URL": "https://huggingface.co/openai/gpt-oss-120b",
+        "Params": 36.2,
+        "Release": "V3",
+        "Thinking": "Reasoning"
+    },
+    {
+        "Model": "Seed-OSS-36B",
+        "Model Type": "General",
+        "Benchmark": "RTLLM",
+        "Task": "Functionality (FNC)",
+        "Result": 61.7,
+        "Model URL": "https://huggingface.co/openai/gpt-oss-120b",
+        "Params": 36.2,
+        "Release": "V3",
+        "Thinking": "Reasoning"
+    },
+    {
+        "Model": "Seed-OSS-36B",
+        "Model Type": "General",
+        "Benchmark": "VerilogEval S2R",
+        "Task": "Synthesis (SYN)",
+        "Result": 70.9,
+        "Model URL": "https://huggingface.co/openai/gpt-oss-120b",
+        "Params": 36.2,
+        "Release": "V3",
+        "Thinking": "Reasoning"
+    },
+    {
+        "Model": "Seed-OSS-36B",
+        "Model Type": "General",
+        "Benchmark": "RTLLM",
+        "Task": "Synthesis (SYN)",
+        "Result": 57.02,
+        "Model URL": "https://huggingface.co/openai/gpt-oss-120b",
+        "Params": 36.2,
+        "Release": "V3",
+        "Thinking": "Reasoning"
+    },
+    {
+        "Model": "Seed-OSS-36B",
+        "Model Type": "General",
+        "Benchmark": "VerilogEval S2R",
+        "Task": "Power",
+        "Result": 70.53,
+        "Model URL": "https://huggingface.co/openai/gpt-oss-120b",
+        "Params": 36.2,
+        "Release": "V3",
+        "Thinking": "Reasoning"
+    },
+    {
+        "Model": "Seed-OSS-36B",
+        "Model Type": "General",
+        "Benchmark": "RTLLM",
+        "Task": "Power",
+        "Result": 65.33,
+        "Model URL": "https://huggingface.co/openai/gpt-oss-120b",
+        "Params": 36.2,
+        "Release": "V3",
+        "Thinking": "Reasoning"
+    },
+    {
+        "Model": "Seed-OSS-36B",
+        "Model Type": "General",
+        "Benchmark": "VerilogEval S2R",
+        "Task": "Performance",
+        "Result": 68.22,
+        "Model URL": "https://huggingface.co/openai/gpt-oss-120b",
+        "Params": 36.2,
+        "Release": "V3",
+        "Thinking": "Reasoning"
+    },
+    {
+        "Model": "Seed-OSS-36B",
+        "Model Type": "General",
+        "Benchmark": "RTLLM",
+        "Task": "Performance",
+        "Result": 57.32,
+        "Model URL": "https://huggingface.co/openai/gpt-oss-120b",
+        "Params": 36.2,
+        "Release": "V3",
+        "Thinking": "Reasoning"
+    },
+    {
+        "Model": "Seed-OSS-36B",
+        "Model Type": "General",
+        "Benchmark": "VerilogEval S2R",
+        "Task": "Area",
+        "Result": 70.33,
+        "Model URL": "https://huggingface.co/openai/gpt-oss-120b",
+        "Params": 36.2,
+        "Release": "V3",
+        "Thinking": "Reasoning"
+    },
+    {
+        "Model": "Seed-OSS-36B",
+        "Model Type": "General",
+        "Benchmark": "RTLLM",
+        "Task": "Area",
+        "Result": 61.39,
+        "Model URL": "https://huggingface.co/openai/gpt-oss-120b",
+        "Params": 36.2,
+        "Release": "V3",
+        "Thinking": "Reasoning"
+    },
+    {
+        "Model": "Seed-OSS-36B",
+        "Model Type": "General",
+        "Benchmark": "RTL-Repo",
+        "Task": "Exact Matching (EM)",
+        "Result": -1.0,
+        "Model URL": "https://huggingface.co/openai/gpt-oss-120b",
+        "Params": 36.2,
+        "Release": "V3",
+        "Thinking": "Reasoning"
+    },
+    {
+        "Model": "Seed-OSS-36B",
+        "Model Type": "General",
+        "Benchmark": "VerilogEval MC",
+        "Task": "Syntax (STX)",
+        "Result": 91.03,
+        "Model URL": "https://huggingface.co/openai/gpt-oss-120b",
+        "Params": 36.2,
+        "Release": "V3",
+        "Thinking": "Reasoning"
+    },
+    {
+        "Model": "Seed-OSS-36B",
+        "Model Type": "General",
+        "Benchmark": "VeriGen",
+        "Task": "Syntax (STX)",
+        "Result": 91.76,
+        "Model URL": "https://huggingface.co/openai/gpt-oss-120b",
+        "Params": 36.2,
+        "Release": "V3",
+        "Thinking": "Reasoning"
+    },
+    {
+        "Model": "Seed-OSS-36B",
+        "Model Type": "General",
+        "Benchmark": "VerilogEval MC",
+        "Task": "Functionality (FNC)",
+        "Result": 73.85,
+        "Model URL": "https://huggingface.co/openai/gpt-oss-120b",
+        "Params": 36.2,
+        "Release": "V3",
+        "Thinking": "Reasoning"
+    },
+    {
+        "Model": "Seed-OSS-36B",
+        "Model Type": "General",
+        "Benchmark": "VeriGen",
+        "Task": "Functionality (FNC)",
+        "Result": 70.59,
+        "Model URL": "https://huggingface.co/openai/gpt-oss-120b",
+        "Params": 36.2,
+        "Release": "V3",
+        "Thinking": "Reasoning"
+    },
+    {
+        "Model": "Seed-OSS-36B",
+        "Model Type": "General",
+        "Benchmark": "VerilogEval MC",
+        "Task": "Synthesis (SYN)",
+        "Result": 73.46,
+        "Model URL": "https://huggingface.co/openai/gpt-oss-120b",
+        "Params": 36.2,
+        "Release": "V3",
+        "Thinking": "Reasoning"
+    },
+    {
+        "Model": "Seed-OSS-36B",
+        "Model Type": "General",
+        "Benchmark": "VeriGen",
+        "Task": "Synthesis (SYN)",
+        "Result": 70.59,
+        "Model URL": "https://huggingface.co/openai/gpt-oss-120b",
+        "Params": 36.2,
+        "Release": "V3",
+        "Thinking": "Reasoning"
+    },
+    {
+        "Model": "Seed-OSS-36B",
+        "Model Type": "General",
+        "Benchmark": "VerilogEval MC",
+        "Task": "Power",
+        "Result": 73.04,
+        "Model URL": "https://huggingface.co/openai/gpt-oss-120b",
+        "Params": 36.2,
+        "Release": "V3",
+        "Thinking": "Reasoning"
+    },
+    {
+        "Model": "Seed-OSS-36B",
+        "Model Type": "General",
+        "Benchmark": "VeriGen",
+        "Task": "Power",
+        "Result": 66.11,
+        "Model URL": "https://huggingface.co/openai/gpt-oss-120b",
+        "Params": 36.2,
+        "Release": "V3",
+        "Thinking": "Reasoning"
+    },
+    {
+        "Model": "Seed-OSS-36B",
+        "Model Type": "General",
+        "Benchmark": "VerilogEval MC",
+        "Task": "Performance",
+        "Result": 70.89,
+        "Model URL": "https://huggingface.co/openai/gpt-oss-120b",
+        "Params": 36.2,
+        "Release": "V3",
+        "Thinking": "Reasoning"
+    },
+    {
+        "Model": "Seed-OSS-36B",
+        "Model Type": "General",
+        "Benchmark": "VeriGen",
+        "Task": "Performance",
+        "Result": 70.19,
+        "Model URL": "https://huggingface.co/openai/gpt-oss-120b",
+        "Params": 36.2,
+        "Release": "V3",
+        "Thinking": "Reasoning"
+    },
+    {
+        "Model": "Seed-OSS-36B",
+        "Model Type": "General",
+        "Benchmark": "VerilogEval MC",
+        "Task": "Area",
+        "Result": 72.6,
+        "Model URL": "https://huggingface.co/openai/gpt-oss-120b",
+        "Params": 36.2,
+        "Release": "V3",
+        "Thinking": "Reasoning"
+    },
+    {
+        "Model": "Seed-OSS-36B",
+        "Model Type": "General",
+        "Benchmark": "VeriGen",
+        "Task": "Area",
+        "Result": 72.45,
+        "Model URL": "https://huggingface.co/openai/gpt-oss-120b",
+        "Params": 36.2,
+        "Release": "V3",
+        "Thinking": "Reasoning"
+    },
     {
         "Model": "QwQ 32B",
         "Model Type": "General",
@@ -3032,7 +3307,7 @@
         "Result": 88.21,
         "Model URL": "https://huggingface.co/openai/gpt-oss-20b",
         "Params": 21.5,
-        "Release": "V2",
+        "Release": "V3",
         "Thinking": "Reasoning"
     },
     {
@@ -3043,7 +3318,7 @@
         "Result": 89.79,
         "Model URL": "https://huggingface.co/openai/gpt-oss-20b",
         "Params": 21.5,
-        "Release": "V2",
+        "Release": "V3",
         "Thinking": "Reasoning"
     },
     {
@@ -3054,7 +3329,7 @@
         "Result": 68.08,
         "Model URL": "https://huggingface.co/openai/gpt-oss-20b",
         "Params": 21.5,
-        "Release": "V2",
+        "Release": "V3",
         "Thinking": "Reasoning"
     },
     {
@@ -3065,7 +3340,7 @@
         "Result": 61.28,
         "Model URL": "https://huggingface.co/openai/gpt-oss-20b",
         "Params": 21.5,
-        "Release": "V2",
+        "Release": "V3",
         "Thinking": "Reasoning"
     },
     {
@@ -3076,7 +3351,7 @@
         "Result": 67.69,
         "Model URL": "https://huggingface.co/openai/gpt-oss-20b",
         "Params": 21.5,
-        "Release": "V2",
+        "Release": "V3",
         "Thinking": "Reasoning"
     },
     {
@@ -3087,7 +3362,7 @@
         "Result": 49.36,
         "Model URL": "https://huggingface.co/openai/gpt-oss-20b",
         "Params": 21.5,
-        "Release": "V2",
+        "Release": "V3",
         "Thinking": "Reasoning"
     },
     {
@@ -3098,7 +3373,7 @@
         "Result": 68.92,
         "Model URL": "https://huggingface.co/openai/gpt-oss-20b",
         "Params": 21.5,
-        "Release": "V2",
+        "Release": "V3",
         "Thinking": "Reasoning"
     },
     {
@@ -3109,7 +3384,7 @@
         "Result": 57.14,
         "Model URL": "https://huggingface.co/openai/gpt-oss-20b",
         "Params": 21.5,
-        "Release": "V2",
+        "Release": "V3",
         "Thinking": "Reasoning"
     },
     {
@@ -3120,7 +3395,7 @@
         "Result": 63.63,
         "Model URL": "https://huggingface.co/openai/gpt-oss-20b",
         "Params": 21.5,
-        "Release": "V2",
+        "Release": "V3",
         "Thinking": "Reasoning"
     },
     {
@@ -3131,7 +3406,7 @@
         "Result": 45.88,
         "Model URL": "https://huggingface.co/openai/gpt-oss-20b",
         "Params": 21.5,
-        "Release": "V2",
+        "Release": "V3",
         "Thinking": "Reasoning"
     },
     {
@@ -3142,7 +3417,7 @@
         "Result": 66.81,
         "Model URL": "https://huggingface.co/openai/gpt-oss-20b",
         "Params": 21.5,
-        "Release": "V2",
+        "Release": "V3",
         "Thinking": "Reasoning"
     },
     {
@@ -3153,7 +3428,7 @@
         "Result": 54.34,
         "Model URL": "https://huggingface.co/openai/gpt-oss-20b",
         "Params": 21.5,
-        "Release": "V2",
+        "Release": "V3",
         "Thinking": "Reasoning"
     },
     {
@@ -3164,7 +3439,7 @@
         "Result": -1.0,
         "Model URL": "https://huggingface.co/openai/gpt-oss-20b",
         "Params": 21.5,
-        "Release": "V2",
+        "Release": "V3",
         "Thinking": "Reasoning"
     },
     {
@@ -3175,7 +3450,7 @@
         "Result": 91.67,
         "Model URL": "https://huggingface.co/openai/gpt-oss-20b",
         "Params": 21.5,
-        "Release": "V2",
+        "Release": "V3",
         "Thinking": "Reasoning"
     },
     {
@@ -3186,7 +3461,7 @@
         "Result": 76.47,
         "Model URL": "https://huggingface.co/openai/gpt-oss-20b",
         "Params": 21.5,
-        "Release": "V2",
+        "Release": "V3",
         "Thinking": "Reasoning"
     },
     {
@@ -3197,7 +3472,7 @@
         "Result": 68.85,
         "Model URL": "https://huggingface.co/openai/gpt-oss-20b",
         "Params": 21.5,
-        "Release": "V2",
+        "Release": "V3",
         "Thinking": "Reasoning"
     },
     {
@@ -3208,7 +3483,7 @@
         "Result": 51.76,
         "Model URL": "https://huggingface.co/openai/gpt-oss-20b",
         "Params": 21.5,
-        "Release": "V2",
+        "Release": "V3",
         "Thinking": "Reasoning"
     },
     {
@@ -3219,7 +3494,7 @@
         "Result": 68.59,
         "Model URL": "https://huggingface.co/openai/gpt-oss-20b",
         "Params": 21.5,
-        "Release": "V2",
+        "Release": "V3",
         "Thinking": "Reasoning"
     },
     {
@@ -3230,7 +3505,7 @@
         "Result": 51.76,
         "Model URL": "https://huggingface.co/openai/gpt-oss-20b",
         "Params": 21.5,
-        "Release": "V2",
+        "Release": "V3",
         "Thinking": "Reasoning"
     },
     {
@@ -3241,7 +3516,7 @@
         "Result": 68.69,
         "Model URL": "https://huggingface.co/openai/gpt-oss-20b",
         "Params": 21.5,
-        "Release": "V2",
+        "Release": "V3",
         "Thinking": "Reasoning"
     },
     {
@@ -3252,7 +3527,7 @@
         "Result": 46.57,
         "Model URL": "https://huggingface.co/openai/gpt-oss-20b",
         "Params": 21.5,
-        "Release": "V2",
+        "Release": "V3",
         "Thinking": "Reasoning"
     },
     {
@@ -3263,7 +3538,7 @@
         "Result": 66.39,
         "Model URL": "https://huggingface.co/openai/gpt-oss-20b",
         "Params": 21.5,
-        "Release": "V2",
+        "Release": "V3",
         "Thinking": "Reasoning"
     },
     {
@@ -3274,7 +3549,7 @@
         "Result": 51.56,
         "Model URL": "https://huggingface.co/openai/gpt-oss-20b",
         "Params": 21.5,
-        "Release": "V2",
+        "Release": "V3",
         "Thinking": "Reasoning"
     },
     {
@@ -3285,7 +3560,7 @@
         "Result": 68.04,
         "Model URL": "https://huggingface.co/openai/gpt-oss-20b",
         "Params": 21.5,
-        "Release": "V2",
+        "Release": "V3",
         "Thinking": "Reasoning"
     },
     {
@@ -3296,7 +3571,7 @@
         "Result": 50.87,
         "Model URL": "https://huggingface.co/openai/gpt-oss-20b",
         "Params": 21.5,
-        "Release": "V2",
+        "Release": "V3",
         "Thinking": "Reasoning"
     },
     {

results/results_verilator_october_2025.csv

@@ -0,0 +1,67 @@
+,Syntax (STX),Syntax (STX),Functionality (FNC),Functionality (FNC),Synthesis (SYN),Synthesis (SYN),Power,Power,Performance,Performance,Area,Area,EM,Syntax (STX),Syntax (STX),Functionality (FNC),Functionality (FNC),Synthesis (SYN),Synthesis (SYN),Power,Power,Performance,Performance,Area,Area
+,VerilogEval S2R,RTLLM,VerilogEval S2R,RTLLM,VerilogEval S2R,RTLLM,VerilogEval S2R,RTLLM,VerilogEval S2R,RTLLM,VerilogEval S2R,RTLLM,RTL-Repo,VerilogEval MC,VeriGen,VerilogEval MC,VeriGen,VerilogEval MC,VeriGen,VerilogEval MC,VeriGen,VerilogEval MC,VeriGen,VerilogEval MC,VeriGen
+DeepSeek R1-0528,96.540000000000006,89.790000000000006,79.099999999999994,67.659999999999997,78.969999999999999,64.260000000000005,78.439999999999998,73.319999999999993,76.290000000000006,65.379999999999995,78.200000000000003,70.859999999999999,-1,94.739999999999995,98.819999999999993,80,80,79.870000000000005,74.120000000000005,79.659999999999997,63.859999999999999,77.280000000000001,74.049999999999997,79.409999999999997,77.010000000000005
+DeepSeek R1,97.689999999999998,93.189999999999998,79.359999999999999,67.659999999999997,79.099999999999994,62.979999999999997,79.010000000000005,71.379999999999995,76.489999999999995,63.740000000000002,78.629999999999995,69.819999999999993,-1,97.439999999999998,95.290000000000006,77.819999999999993,74.120000000000005,77.689999999999998,74.120000000000005,77.299999999999997,64.579999999999998,75.030000000000001,74.420000000000002,76.939999999999998,77.150000000000006
+Qwen3 Coder 480B A35B,95.900000000000006,89.359999999999999,66.030000000000001,61.280000000000001,65.260000000000005,54.039999999999999,65.329999999999998,53.43,62.450000000000003,53.43,64.239999999999995,52.259999999999998,38.880000000000003,83.969999999999999,100,58.719999999999999,75.290000000000006,58.210000000000001,75.290000000000006,57.579999999999998,69.629999999999995,55.420000000000002,73.950000000000003,56.5,76.760000000000005
+Llama 3.1 405B,88.209999999999994,83.400000000000006,56.789999999999999,50.210000000000001,56.409999999999997,43.829999999999998,55.159999999999997,53.869999999999997,54.009999999999998,33.829999999999998,53.990000000000002,45.729999999999997,34.619999999999997,90.510000000000005,95.290000000000006,57.689999999999998,60,56.670000000000002,55.289999999999999,55.950000000000003,51.57,54.090000000000003,54.869999999999997,55.259999999999998,57.670000000000002
+Qwen3 236B A22B,93.969999999999999,78.719999999999999,75.769999999999996,56.170000000000002,75.379999999999995,50.210000000000001,75,57.009999999999998,72.939999999999998,47.850000000000001,74.780000000000001,52.450000000000003,41.939999999999998,83.719999999999999,88.239999999999995,68.969999999999999,62.350000000000001,68.459999999999994,62.350000000000001,68.299999999999997,54.630000000000003,65.989999999999995,62.630000000000003,68.349999999999994,62.780000000000001
+gpt-oss-120b,94.359999999999999,85.959999999999994,76.670000000000002,62.130000000000003,76.540000000000006,51.490000000000002,76.510000000000005,53.960000000000001,73.829999999999998,49.850000000000001,76.010000000000005,54.640000000000001,-1,95.379999999999995,89.409999999999997,76.920000000000002,69.409999999999997,76.670000000000002,69.409999999999997,76.480000000000004,71.049999999999997,73.909999999999997,69.439999999999998,75.810000000000002,70.879999999999995
+Qwen2.5 72B,83.849999999999994,82.980000000000004,55,48.509999999999998,54.490000000000002,45.960000000000001,53.969999999999999,48.450000000000003,51.299999999999997,47.340000000000003,52.840000000000003,49.68,37.439999999999998,83.079999999999998,87.060000000000002,54.740000000000002,50.590000000000003,54.359999999999999,50.590000000000003,53.359999999999999,50.049999999999997,51.200000000000003,50.560000000000002,52.759999999999998,52.229999999999997
+Llama 3.(1-3) 70B,68.329999999999998,79.150000000000006,40.899999999999999,45.960000000000001,40.899999999999999,40.43,40.770000000000003,42.119999999999997,38.329999999999998,39.340000000000003,40.420000000000002,40.920000000000002,28.719999999999999,86.030000000000001,87.060000000000002,43.719999999999999,63.530000000000001,43.590000000000003,63.530000000000001,43.549999999999997,62.289999999999999,41.57,63.520000000000003,42.939999999999998,63.539999999999999
+Seed-OSS-36B,89.359999999999999,83.400000000000006,71.150000000000006,61.700000000000003,70.900000000000006,57.020000000000003,70.530000000000001,65.329999999999998,68.219999999999999,57.32,70.329999999999998,61.390000000000001,-1,91.030000000000001,91.760000000000005,73.849999999999994,70.590000000000003,73.459999999999994,70.590000000000003,73.040000000000006,66.109999999999999,70.890000000000001,70.189999999999998,72.599999999999994,72.450000000000003
+QwQ 32B,89.230000000000004,85.959999999999994,67.560000000000002,58.299999999999997,67.180000000000007,53.619999999999997,67.040000000000006,57.289999999999999,64.780000000000001,53.340000000000003,66.439999999999998,57.560000000000002,-1,59.619999999999997,87.060000000000002,40.130000000000003,65.879999999999995,39.490000000000002,64.709999999999994,39.259999999999998,60.950000000000003,37.490000000000002,64.480000000000004,39.170000000000002,66.310000000000002
+Qwen2.5 32B,90.900000000000006,87.659999999999997,55.899999999999999,50.210000000000001,55.259999999999998,46.810000000000002,55.560000000000002,50.659999999999997,53.100000000000001,47.409999999999997,54.020000000000003,51.369999999999997,28.93,95.510000000000005,85.879999999999995,45,49.409999999999997,43.719999999999999,49.409999999999997,43.719999999999999,46.409999999999997,42.490000000000002,49.890000000000001,42.950000000000003,49.609999999999999
+Magistral Small 2506,65.129999999999995,74.040000000000006,42.950000000000003,44.259999999999998,42.310000000000002,37.450000000000003,41.810000000000002,43.100000000000001,40.189999999999998,37.369999999999997,41.75,40.350000000000001,-1,31.789999999999999,70.590000000000003,21.030000000000001,55.289999999999999,20.77,54.119999999999997,20.329999999999998,46.600000000000001,20.079999999999998,54.020000000000003,20.66,55.060000000000002
+gpt-oss-20b,88.209999999999994,89.790000000000006,68.079999999999998,61.280000000000001,67.689999999999998,49.359999999999999,68.920000000000002,57.140000000000001,63.630000000000003,45.880000000000003,66.810000000000002,54.340000000000003,-1,91.670000000000002,76.469999999999999,68.849999999999994,51.759999999999998,68.590000000000003,51.759999999999998,68.689999999999998,46.57,66.390000000000001,51.560000000000002,68.040000000000006,50.869999999999997
+StarChat2 15B v0.1,88.719999999999999,89.359999999999999,38.850000000000001,48.509999999999998,38.850000000000001,45.960000000000001,38.920000000000002,48.950000000000003,36.18,45.049999999999997,38.810000000000002,48.630000000000003,13.42,81.409999999999997,91.760000000000005,36.539999999999999,63.530000000000001,36.280000000000001,63.530000000000001,36.189999999999998,57.509999999999998,35.109999999999999,63.229999999999997,36.240000000000002,65
+DeepSeek R1 Distill Qwen 14B,42.82,36.600000000000001,25.260000000000002,20.43,25,17.02,24.809999999999999,18.620000000000001,23.68,17.18,24.829999999999998,18.07,-1,45.640000000000001,44.710000000000001,25.379999999999999,29.41,25.260000000000002,29.41,24.98,25.920000000000002,23.609999999999999,29.43,24.690000000000001,29.649999999999999
+CodeLlama 70B,67.310000000000002,77.450000000000003,34.740000000000002,38.719999999999999,34.490000000000002,36.170000000000002,34.030000000000001,39.18,32.880000000000003,36.140000000000001,33.829999999999998,37.93,24.329999999999998,92.560000000000002,88.239999999999995,35.770000000000003,52.939999999999998,35.380000000000003,51.759999999999998,34.810000000000002,50.619999999999997,32.340000000000003,51.890000000000001,34.520000000000003,52.359999999999999
+DeepSeek Coder 33B,64.489999999999995,88.939999999999998,23.59,46.380000000000003,23.59,44.259999999999998,23.370000000000001,44.539999999999999,23.32,41.539999999999999,22.739999999999998,45.039999999999999,24.579999999999998,76.540000000000006,88.239999999999995,40.130000000000003,50.590000000000003,39.869999999999997,50.590000000000003,38.700000000000003,51.5,37.329999999999998,50.219999999999999,38.399999999999999,51.630000000000003
+QwenCoder 2.5 32B,90.260000000000005,81.280000000000001,46.539999999999999,47.659999999999997,46.409999999999997,45.109999999999999,45.950000000000003,48.810000000000002,44.420000000000002,45.039999999999999,45.57,47.359999999999999,31.07,86.150000000000006,85.879999999999995,48.210000000000001,54.119999999999997,47.560000000000002,54.119999999999997,47.350000000000001,54.590000000000003,45.350000000000001,54.119999999999997,46.780000000000001,55.140000000000001
+DeepCoder 14B,45.130000000000003,42.549999999999997,28.719999999999999,25.530000000000001,28.719999999999999,22.98,28.579999999999998,26.059999999999999,26.859999999999999,22.940000000000001,28.16,24.140000000000001,-1,63.460000000000001,49.409999999999997,34.359999999999999,37.649999999999999,34.100000000000001,35.289999999999999,34.170000000000002,32.789999999999999,32.579999999999998,35.289999999999999,33.409999999999997,35.280000000000001
+QwenCoder 2.5 14B,80.900000000000006,82.980000000000004,39.869999999999997,50.640000000000001,38.969999999999999,47.659999999999997,37.490000000000002,47.82,36.369999999999997,46.979999999999997,36.729999999999997,48.399999999999999,37.530000000000001,81.409999999999997,84.709999999999994,43.079999999999998,50.590000000000003,42.439999999999998,50.590000000000003,42.020000000000003,49.509999999999998,40.479999999999997,50.859999999999999,42.109999999999999,50.149999999999999
+SeedCoder 8B,96.409999999999997,89.359999999999999,54.359999999999999,51.490000000000002,54.229999999999997,48.509999999999998,53.68,51.520000000000003,51.5,47.539999999999999,52.560000000000002,51.700000000000003,28.23,78.209999999999994,91.760000000000005,38.210000000000001,47.060000000000002,38.079999999999998,43.530000000000001,38.100000000000001,37.890000000000001,36.090000000000003,43.770000000000003,37.539999999999999,43.869999999999997
+SeedCoder 8B Reasoning,68.719999999999999,56.170000000000002,48.079999999999998,34.039999999999999,48.079999999999998,30.640000000000001,48.270000000000003,33.409999999999997,45.780000000000001,29.859999999999999,47,31.27,-1,84.489999999999995,80,48.969999999999999,63.530000000000001,48.850000000000001,63.530000000000001,48.68,54.979999999999997,46.32,62.969999999999999,47.530000000000001,66.099999999999994
+OpenCoder 8B,80,80,27.949999999999999,43.829999999999998,27.949999999999999,42.130000000000003,27.52,42.439999999999998,26.07,40.979999999999997,27.280000000000001,42.460000000000001,16.170000000000002,81.030000000000001,94.120000000000005,36.789999999999999,49.409999999999997,36.149999999999999,44.710000000000001,35.890000000000001,41.219999999999999,34.240000000000002,44.579999999999998,35.899999999999999,45.079999999999998
+QwenCoder 2.5 7B,19.23,83.400000000000006,7.1799999999999997,40.850000000000001,7.1799999999999997,38.719999999999999,7.0199999999999996,42.310000000000002,7.1500000000000004,38.710000000000001,7.0800000000000001,41.200000000000003,28.329999999999998,76.150000000000006,90.590000000000003,35,48.240000000000002,34.740000000000002,48.240000000000002,34.619999999999997,44.93,32.799999999999997,48.399999999999999,34.649999999999999,49.759999999999998
+DeepSeek Coder 6.7B,86.030000000000001,89.359999999999999,31.280000000000001,45.960000000000001,31.149999999999999,42.979999999999997,31.16,47.369999999999997,29.149999999999999,43.299999999999997,31.079999999999998,48.170000000000002,24.629999999999999,69.099999999999994,83.530000000000001,31.670000000000002,37.649999999999999,30.129999999999999,37.649999999999999,30.190000000000001,36.450000000000003,28.16,37.469999999999999,29.82,37.659999999999997
+CodeV R1 Distill Qwen 7B,56.409999999999997,79.150000000000006,33.969999999999999,50.640000000000001,33.850000000000001,48.939999999999998,33.799999999999997,50.920000000000002,32.490000000000002,49.039999999999999,33.710000000000001,50.670000000000002,-1,94.230000000000004,87.060000000000002,22.18,63.530000000000001,22.18,63.530000000000001,22.210000000000001,57.75,20.73,63.240000000000002,22.079999999999998,63.890000000000001
+HaVen-CodeQwen,93.969999999999999,84.260000000000005,46.789999999999999,47.229999999999997,46.280000000000001,44.259999999999998,45.469999999999999,42.270000000000003,44.289999999999999,42.07,45.5,44.270000000000003,25.140000000000001,95,95.290000000000006,50,67.060000000000002,50,67.060000000000002,48.509999999999998,65.700000000000003,47.560000000000002,67.609999999999999,48.140000000000001,66.799999999999997
+CodeV-QW-7B,45.259999999999998,71.489999999999995,20.640000000000001,36.170000000000002,20,29.359999999999999,19.940000000000001,30.309999999999999,19.739999999999998,24.52,19.879999999999999,28.48,20.940000000000001,92.689999999999998,96.469999999999999,51.539999999999999,57.649999999999999,51.409999999999997,57.649999999999999,51.520000000000003,56.439999999999998,48.710000000000001,57.560000000000002,51.310000000000002,58.350000000000001
+RTLCoder Mistral,56.030000000000001,35.32,25.379999999999999,17.02,25.379999999999999,16.170000000000002,25.52,15.859999999999999,23.18,15.34,25.539999999999999,15.84,14.77,62.310000000000002,87.060000000000002,27.309999999999999,56.469999999999999,27.18,56.469999999999999,26.93,52.350000000000001,25.43,56.549999999999997,26.949999999999999,59.170000000000002
+RTLCoder DeepSeek,84.739999999999995,76.170000000000002,40.509999999999998,42.130000000000003,40.380000000000003,37.450000000000003,39.729999999999997,36.75,38.439999999999998,35.289999999999999,39.399999999999999,36.390000000000001,19.350000000000001,79.230000000000004,85.879999999999995,36.409999999999997,55.289999999999999,36.409999999999997,55.289999999999999,36.490000000000002,50.789999999999999,34.509999999999998,55.219999999999999,36.210000000000001,54.689999999999998
+OriGen,96.790000000000006,85.109999999999999,54.359999999999999,53.189999999999998,53.850000000000001,51.060000000000002,54.079999999999998,54.170000000000002,51.07,51.310000000000002,53.009999999999998,54.390000000000001,17.07,92.560000000000002,96.469999999999999,50.509999999999998,74.120000000000005,50.509999999999998,74.120000000000005,50.829999999999998,69.299999999999997,48.280000000000001,74.260000000000005,50.409999999999997,76.349999999999994
+CodeV-CL-7B,32.82,54.469999999999999,13.460000000000001,29.359999999999999,13.33,22.550000000000001,13.19,23.190000000000001,13.26,21.469999999999999,13.19,22.190000000000001,12.27,92.180000000000007,98.819999999999993,31.789999999999999,49.409999999999997,31.789999999999999,49.409999999999997,31.739999999999998,48.130000000000003,29.829999999999998,49.340000000000003,31.609999999999999,49.079999999999998
+CodeV-DS-6.7B,34.359999999999999,69.790000000000006,14.74,41.280000000000001,14.359999999999999,37.869999999999997,14.380000000000001,35.890000000000001,14.130000000000001,35.740000000000002,14.19,36.18,21.260000000000002,95.769999999999996,100,46.670000000000002,61.18,46.670000000000002,61.18,47.07,61.060000000000002,44.189999999999998,61.130000000000003,46.469999999999999,60.950000000000003
+,,,,,,,,,,,,,,,,,,,,,,,,,
+,,,,,,,,,,,,,,,,,,,,,,,,,
+,,,,,,,,,,,,,,,,,,,,,,,,,
+,,,,,,,,,,,,,,,,,,,,,,,,,
+,,,,,,,,,,,,,,,,,,,,,,,,,
+,,,,,,,,,,,,,,,,,,,,,,,,,
+,,,,,,,,,,,,,,,,,,,,,,,,,
+,,,,,,,,,,,,,,,,,,,,,,,,,
+,,,,,,,,,,,,,,,,,,,,,,,,,
+,,,,,,,,,,,,,,,,,,,,,,,,,
+,,,,,,,,,,,,,,,,,,,,,,,,,
+,,,,,,,,,,,,,,,,,,,,,,,,,
+,,,,,,,,,,,,,,,,,,,,,,,,,
+,,,,,,,,,,,,,,,,,,,,,,,,,
+,,,,,,,,,,,,,,,,,,,,,,,,,
+,,,,,,,,,,,,,,,,,,,,,,,,,
+,,,,,,,,,,,,,,,,,,,,,,,,,
+,,,,,,,,,,,,,,,,,,,,,,,,,
+,,,,,,,,,,,,,,,,,,,,,,,,,
+,,,,,,,,,,,,,,,,,,,,,,,,,
+,,,,,,,,,,,,,,,,,,,,,,,,,
+,,,,,,,,,,,,,,,,,,,,,,,,,
+,,,,,,,,,,,,,,,,,,,,,,,,,
+,,,,,,,,,,,,,,,,,,,,,,,,,
+,,,,,,,,,,,,,,,,,,,,,,,,,
+,,,,,,,,,,,,,,,,,,,,,,,,,
+,,,,,,,,,,,,,,,,,,,,,,,,,
+,,,,,,,,,,,,,,,,,,,,,,,,,
+,,,,,,,,,,,,,,,,,,,,,,,,,
+,,,,,,,,,,,,,,,,,,,,,,,,,
+,,,,,,,,,,,,,,,,,,,,,,,,,
+