Upload 8 files

_about_us.md

@@ -0,0 +1,14 @@
+## About Us
+
+
+#### Contact:
+
+[Jingcong Liang](https://github.com/ljcleo), [Shijun Wan](https://github.com/JunsWan), and [Xuehai Wu]().
+
+[Siyuan Wang]()
+
+
+### Contact
+
+Please contact us in the following ways:
+

_header.md

@@ -0,0 +1,51 @@
+<br/>
+
+# HardcoreLogic: Challenging Large Reasoning Models with Long-tail Logic Puzzle Games
+
+<div style="text-align: center; margin-top: 20px; margin-bottom: 10px;">
+  <a href="#" class="btn-glass btn-paper">📄 Paper</a>
+  <a href="https://github.com/ljcleo/hardcore-logic" class="btn-glass btn-github">💻 GitHub</a>
+  <a href="https://huggingface.co/datasets/xhWu-fd/HardcoreLogic" class="btn-glass btn-dataset">🤗 Dataset</a>
+</div>
+
+<div style="text-align: center; font-size: 14px; color: #555; margin-top: 8px;">
+  Last Updated: <b>{LAST_UPDATED}</b>
+</div>
+
+<style>
+.btn-glass {
+    display: inline-block;
+    padding: 12px 25px;
+    margin: 8px;
+    font-weight: 500;
+    text-decoration: none !important; /* 去掉下划线 */
+    border-radius: 12px;
+    color: white;
+    backdrop-filter: blur(6px); /* 虚化效果 */
+    background: rgba(255,255,255,0.85); /* 半透明，alpha 调高 */
+    box-shadow: 0 4px 15px rgba(0,0,0,0.2);
+    transition: all 0.3s ease;
+}
+
+/* 默认按钮颜色 */
+.btn-paper { background-color: rgba(0, 123, 255, 0.85); }
+.btn-github { background-color: rgba(40, 167, 69, 0.85); }
+.btn-dataset { background-color: rgba(255, 183, 3, 0.85); }
+
+/* hover 渐变 + 悬浮效果 */
+.btn-paper:hover { 
+    background: linear-gradient(135deg, rgba(0,123,255,0.95), rgba(0,198,255,0.95));
+    box-shadow: 0 10px 20px rgba(0,0,0,0.35);
+    transform: translateY(-3px);
+}
+.btn-github:hover { 
+    background: linear-gradient(135deg, rgba(27,94,32,0.95), rgba(40,167,69,0.95));
+    box-shadow: 0 10px 20px rgba(0,0,0,0.35);
+    transform: translateY(-3px);
+}
+.btn-dataset:hover { 
+    background: linear-gradient(135deg, rgba(255,140,0,0.95), rgba(255,183,3,0.95));
+    box-shadow: 0 10px 20px rgba(0,0,0,0.35);
+    transform: translateY(-3px);
+}
+</style>

constants.py

@@ -0,0 +1,322 @@
+from collections import OrderedDict
+
+BANNER = '''
+<div style="
+    display: flex;
+    justify-content: center;
+    align-items: center;
+    height: 120px;
+    background: #ffffff;  
+    border-bottom: 2px solid #ced4da;
+">
+  <h1 style="
+      font-size: 5em;
+      font-family: 'Poppins', 'Segoe UI', sans-serif;
+      background: linear-gradient(90deg, #007bff, #00c6ff, #0096c7, #0077b6);
+      -webkit-background-clip: text;
+      color: transparent;
+      animation: gradientFlow 4s ease-in-out infinite;
+      background-size: 300%;
+      letter-spacing: 2px;
+  ">
+    HardcoreLogic
+  </h1>
+</div>
+
+<style>
+@keyframes gradientFlow {
+  0% { background-position: 0% 50%; }
+  50% { background-position: 100% 50%; }
+  100% { background-position: 0% 50%; }
+}
+</style>
+'''
+
+
+CITATION_TEXT = """
+
+"""
+'''
+   @article{hardcorelogic2025,
+    title={HardcoreLogic: Challenging Large Reasoning Models with Long-tail Logic Puzzle Games},
+    author={},
+    year={2025},
+    url={https://arxiv.org/},
+}'''
+
+column_names = OrderedDict({
+    "model": "Model",
+    "open-source": "Open Source",
+    "total accuracy": "Total Acc",
+    "unsolvable puzzle": "Unsolvable Puzzle ACC",
+})
+
+column_names_puzzle = OrderedDict({
+    "model": "Model",
+    "total accuracy": "Total Acc",
+    "Zebra": "Zebra",
+    "Binario": "Binario",
+    "Crypto": "Crypto",
+    "Hanoi": "Hanoi",
+    "Hitpri": "Hitpri",
+    "Kakurasu": "Kakurasu",
+    "Minesweeper": "Minesweeper",
+    "Navigation": "Navigation",
+    "Skyscraper": "Skyscraper",
+    "Sudoku": "Sudoku",
+})
+
+LEADERBOARD_REMARKS = """**WB Reward**: for each comparison (A vs B), a reward for A is **+/-1** if A is **much better/worse** than B, and **+/-0.5** if A is **slightly better/worse** than B; when there is a **Tie**, the reward is **0**.
+"""
+
+LEADERBOARD_REMARKS_MAIN = """
+"""
+
+RANKING_COLUMN = "total accuracy"
+
+ORDERED_COLUMN_NAMES = [
+    "model",
+    "mode",
+    "open-source",
+    "total accuracy",
+    "increased complexity",
+    "uncommon elements",
+    "unsolvable puzzle",
+    "temperature",
+    "n_sampling",
+    "n"
+]
+
+ORDERED_COLUMN_NAMES_PUZZLE = [
+    "model",
+    "mode",
+    "open-source",
+    "total accuracy",
+    "Zebra",
+    "Binario",
+    "Crypto",
+    "Hanoi",
+    "Hitpri",
+    "Kakurasu",
+    "Minesweeper",
+    "Navigation",
+    "Skyscraper",
+    "Sudoku",
+    "temperature",
+    "n_sampling",
+    "n"
+]
+
+
+js_light = """
+function refresh() {
+    const url = new URL(window.location);
+
+    if (url.searchParams.get('__theme') !== 'light') {
+        url.searchParams.set('__theme', 'light');
+        window.location.href = url.href;
+    }
+
+    // Find the fieldset with the given id
+    const fieldset = document.getElementById("rank-column-radio");
+
+    // Create a new span element with the text "Decoding Mode:"
+    const rankBySpan = document.createElement("span");
+    rankBySpan.textContent = "Decoding Mode: ";
+    rankBySpan.style.fontWeight = "bold"; // Optional: make the text bold
+     rankBySpan.style.fontSize = "19px"; // Larger font size
+    rankBySpan.style.paddingRight = "18px"; // Add padding on the right
+
+    // Wrap the span and the labels in a flex container
+    const flexContainer = document.createElement("div");
+    flexContainer.style.display = "flex";
+    flexContainer.style.alignItems = "center";
+
+    // Insert the rankBySpan at the beginning of the flex container
+    flexContainer.appendChild(rankBySpan);
+
+    // Move all existing labels into the flex container
+    while (fieldset.firstChild) {
+        flexContainer.appendChild(fieldset.firstChild);
+    }
+
+    // Append the flex container back to the fieldset
+    fieldset.appendChild(flexContainer);
+}
+"""
+
+js_code = """
+function scroll_top() {
+    console.log("Hello from Gradio!");
+    const bubbles = document.querySelectorAll('.bubble-wrap');
+    bubbles.forEach((bubble, index) => {
+        setTimeout(() => {
+            bubble.scrollTop = 0;
+        }, index * 100); // Delay of 100ms between each iteration
+    });
+
+}
+"""
+
+
+TASK_TYPE_STR = "**Tasks**: Info seeking (**InfoSek**), Creative Writing (**CrtWrt**), Coding&Debugging (**Code**), Reasoning (**Reason**), Editing (**Edit**), **Math**, Planning (**Plan**), Brainstorming (**Brnstrm**), Role playing (**RolPly**), Advice seeking (**AdvSek**), Data Analysis (**DataAna**)"
+
+css = """
+/* ========== 🌟 Global Typography ========== */
+code {
+    font-size: large;
+}
+footer {visibility: hidden}
+
+.markdown-text{font-size: 14pt}
+.markdown-text-small{font-size: 13pt}
+.markdown-text-tiny{font-size: 12pt}
+
+/* ========== 🎓 Fudan Blue Theme Colors ========== */
+:root {
+    --fudan-blue: #002D72;
+    --fudan-blue-light: #E6EEF8;
+    --fudan-gray: #f7f7f7;
+    --fudan-border: #c8d6e5;
+    --fudan-highlight: #1E56A0;
+}
+
+/* ========== 🏅 Leaderboard Table ========== */
+#leaderboard-table th,
+#leaderboard-puzzle-table th {
+    background-color: var(--fudan-blue);
+    color: white;
+    text-align: center;
+    padding: 10px;
+    font-size: 15px;
+    border-bottom: 2px solid var(--fudan-border);
+}
+
+#leaderboard-table td,
+#leaderboard-puzzle-table td {
+    text-align: center;
+    font-size: 14px;
+    padding: 8px;
+    background-color: white;
+    border-bottom: 1px solid var(--fudan-border);
+}
+
+/* Hover 行高亮 */
+#leaderboard-table tr:hover,
+#leaderboard-puzzle-table tr:hover {
+    background-color: var(--fudan-blue-light);
+    transition: background-color 0.2s ease-in-out;
+}
+
+/* 表格整体外观 */
+#leaderboard-table,
+#leaderboard-puzzle-table {
+    border-collapse: collapse;
+    border-radius: 10px;
+    overflow: hidden;
+    box-shadow: 0 0 10px rgba(0, 45, 114, 0.15);
+}
+
+/* ========== 📊 Tabs ========== */
+.tab-buttons button[role="tab"] {
+    font-size: 15px;
+    font-weight: 600;
+    color: var(--fudan-blue);
+    border: 1px solid var(--fudan-border);
+    border-radius: 8px;
+    background-color: white;
+    padding: 8px 16px;
+    margin-right: 5px;
+    transition: all 0.2s ease-in-out;
+}
+
+.tab-buttons button[role="tab"]:hover {
+    background-color: var(--fudan-blue-light);
+}
+
+button.selected[role="tab"][aria-selected="true"] {
+    background-color: var(--fudan-blue);
+    color: white;
+    font-weight: bold;
+    font-size: 16px;
+}
+
+/* ========== 📦 Accordion & Buttons ========== */
+.accordion-label button span{
+    font-size: 14pt;
+    font-weight: bold;
+    color: var(--fudan-blue);
+}
+
+.btn_boderline{
+    border: 1px solid var(--fudan-blue);
+    border-radius: 5px;
+    padding: 6px 12px;
+    margin: 5px;
+    font-size: 14pt;
+    font-weight: bold;
+    background-color: var(--fudan-blue-light);
+    color: var(--fudan-blue);
+    transition: background-color 0.3s;
+}
+.btn_boderline:hover{
+    background-color: var(--fudan-blue);
+    color: white;
+}
+
+/* ========== 🧩 Box & Card ========== */
+.box_md{
+    border: 1px solid var(--fudan-border);
+    border-radius: 10px;
+    padding: 10px;
+    font-size: 12pt;
+    margin: 8px;
+    background-color: white;
+    box-shadow: 0 0 6px rgba(0, 45, 114, 0.1);
+}
+
+/* ========== 💬 Markdown Text Enhancements ========== */
+.markdown-text-details{
+    margin: 10px;
+    padding: 10px;
+    background-color: var(--fudan-gray);
+    border-left: 4px solid var(--fudan-blue);
+    border-radius: 6px;
+}
+
+/* ========== 📈 Plot & Visualization ========== */
+.plotly-plot{
+    height: auto;
+    max-height: 600px;
+    min-height: 600px;
+    border: 1px solid var(--fudan-border);
+    border-radius: 10px;
+}
+
+/* ========== 🧷 Misc Components ========== */
+.sample_button{
+    border: 2px solid var(--fudan-blue);
+    border-radius: 10px;
+    padding: 10px;
+    font-size: 17pt;
+    font-weight: bold;
+    margin: 5px;
+    background-color: var(--fudan-blue-light);
+    color: var(--fudan-blue);
+    transition: all 0.3s ease-in-out;
+}
+.sample_button:hover {
+    background-color: var(--fudan-blue);
+    color: white;
+}
+
+/* Scrollable Table Containers */
+#leaderboard-table,
+#leaderboard-puzzle-table {
+    display: block;
+    max-height: 800px;
+    overflow-y: auto;
+}
+
+
+"""

data_utils.py

@@ -0,0 +1,33 @@
+from datasets.utils.logging import disable_progress_bar
+from constants import column_names, RANKING_COLUMN, ORDERED_COLUMN_NAMES
+from utils_display import make_clickable_model
+disable_progress_bar()
+import json
+import os
+
+summary_file = "HardcoreLogic-Eval/results_dirs/hardcorelogic.summary.json"
+result_dir = "HardcoreLogic-Eval/results_dirs"
+results_by_model = {}
+
+# Formats the columns
+def formatter(x):
+    if type(x) is str:
+        x = x
+    else: 
+        x = round(x, 2)
+    return x
+
+def post_processing(df, column_names, rank_column=RANKING_COLUMN, ordered_columns=ORDERED_COLUMN_NAMES, click_url=True):
+    df = df[[col for col in column_names.keys() if col in df.columns]].copy()
+    for col in df.columns:
+        if col == "model" and click_url:
+            df[col] = df[col].apply(lambda x: x.replace(x, make_clickable_model(x)))
+        else:
+            df[col] = df[col].apply(formatter) # For numerical values
+    list_columns = [col for col in ordered_columns if col in df.columns]
+    df = df[list_columns]
+    if rank_column in df.columns:
+        df.sort_values(by=rank_column, inplace=True, ascending=False)
+    df.rename(columns=column_names, inplace=True)
+    return df
+

model_info.json

@@ -0,0 +1,23 @@
+{
+  "Qwen3-8B": {"pretty_name": "Qwen3-8B","hf_model_id": "Qwen/Qwen3-8B"},
+  "Qwen3-30B-A3B-Thinking-2507": {"pretty_name": "Qwen3-30B-A3B-Thinking-2507","hf_model_id": "Qwen/Qwen3-30B-A3B-Thinking-2507"},
+  "Qwen3-32B": {"pretty_name": "Qwen3-32B","hf_model_id": "Qwen/Qwen3-32B"},
+  "Qwen3-Next-80B-A3B-Thinking": {"pretty_name": "Qwen3-Next-80B-A3B-Thinking","hf_model_id": "Qwen/QQwen3-Next-80B-A3B-Thinking"},
+  "Qwen3-235B-A22B-Thinking-2507": {"pretty_name": "Qwen3-235B-A22B-Thinking-2507","hf_model_id": "Qwen/Qwen3-235B-A22B-Thinking-2507"},
+  "MiniMax-M1-40k": {"pretty_name": "MiniMax-M1-40k","hf_model_id": "MiniMaxAI/MiniMax-M1-40k"},
+  "DeepSeek-R1-0528-Qwen3-8B": {"pretty_name": "DeepSeek-R1-0528-Qwen3-8B","hf_model_id": "deepseek-ai/DeepSeek-R1-0528-Qwen3-8B"},
+  "DeepSeek-V3.1": {"pretty_name": "DeepSeek-V3.1","hf_model_id": "deepseek-ai/DeepSeek-V3.1"},
+  "DeepSeek-R1-0528": {"pretty_name": "DeepSeek-R1-0528","hf_model_id": "deepseek-ai/DeepSeek-R1-0528"},
+  "GLM-4.5": {"pretty_name": "GLM-4.5","hf_model_id": "zai-org/GLM-4.5"},
+  "Kimi-K2-Instruct": {"pretty_name": "Kimi-K2-Instruct","hf_model_id": "moonshotai/Kimi-K2-Instruct"},
+  "Seed-OSS-36B-Instruct": {"pretty_name": "Seed-OSS-36B-Instruct","hf_model_id": "ByteDance-Seed/Seed-OSS-36B-Instruct"},
+  "gpt-oss-120b": {"pretty_name": "gpt-oss-120b","hf_model_id": "openai/gpt-oss-120b"},
+  "gpt-5": {"pretty_name": "gpt-5","hf_model_id": false},
+  "gpt-5-mini": {"pretty_name": "gpt-5-mini","hf_model_id": false},
+  "o4-mini": {"pretty_name": "o4-mini","hf_model_id": false},
+  "grok-4": {"pretty_name": "grok-4","hf_model_id": false},
+  "gemini-2.5-pro": {"pretty_name": "gemini-2.5-pro","hf_model_id": false},
+  "grok-3-mini": {"pretty_name": "grok-3-mini","hf_model_id": false},
+  "claude-sonnet-4-thinking": {"pretty_name": "claude-sonnet-4-thinking","hf_model_id": false},
+  "gemini-2.5-flash": {"pretty_name": "gemini-2.5-flash","hf_model_id": false}
+}

requirements.txt

@@ -0,0 +1,5 @@
+gradio[oauth]==4.19.2
+datasets
+toolz==0.12.1
+plotly
+tabulate

utils_display.py

@@ -0,0 +1,32 @@
+import json 
+
+with open("model_info.json", "r") as f:
+    model_info = json.load(f)
+
+def make_clickable_model(model_name):
+    global model_info
+    modified_model_name = model_name
+
+    if model_name in model_info:
+        info = model_info[model_name]
+        pretty_name = info.get("pretty_name", model_name)
+        hf_id = info.get("hf_model_id", False)
+
+        if hf_id:
+            link = f"https://huggingface.co/{hf_id}"
+            modified_model_name = f'<a target="_blank" href="{link}" style="color: var(--link-text-color); text-decoration: underline; text-decoration-style: dotted;">{pretty_name}</a>'
+        else:
+            modified_model_name = f'{pretty_name}'
+
+    return modified_model_name
+
+    
+
+def styled_error(error):
+    return f"<p style='color: red; font-size: 20px; text-align: center;'>{error}</p>"
+
+def styled_warning(warn):
+    return f"<p style='color: orange; font-size: 20px; text-align: center;'>{warn}</p>"
+
+def styled_message(message):
+    return f"<p style='color: green; font-size: 20px; text-align: center;'>{message}</p>"