import argparse import gradio as gr import pandas as pd import json from constants import * from datetime import datetime from utils_display import model_info from constants import column_names import pytz from data_utils import post_processing LAST_UPDATED = None INTRO_MD = "" with open("_about_us.md", "r") as f: ABOUT_MD = f.read() with open("_header.md", "r") as f: HEADER_MD = f.read() raw_data = None original_df = None raw_puzzle_data = None puzzle_df = None available_models = list(model_info.keys()) def _gstr(text): return gr.Text(text, visible=False) def _tab_leaderboard(): global original_df df =original_df.copy() df.insert(0, "#", range(1, 1 + len(df))) if "Open Source" in df.columns: df["Open Source"] = df["Open Source"].apply(lambda x: "✅" if x else "❌") leaderboard_table = gr.components.Dataframe( value=df, datatype=["number", "markdown", "bool", "number", "number", "number", "number"], elem_id="leaderboard-table", interactive=False, visible=True, column_widths=[50, 200, 100, 120, 120, 120, 130], wrap=True, height=800 ) return leaderboard_table def _tab_leaderboard_puzzle(): global puzzle_df df =puzzle_df.copy() df.insert(0, "#", range(1, 1 + len(df))) leaderboard_puzzle_table = gr.components.Dataframe( value=df, datatype=["number", "markdown", "number", "number", "number", "number", "number", "number", "number", "number", "number", "number"], elem_id="leaderboard-puzzle-table", interactive=False, visible=True, column_widths=[50, 200, 150, 150, 150, 150, 150, 150, 150, 150, 150,150, 150], wrap=True, height=800 ) return leaderboard_puzzle_table def _tab_submit(): markdown_text = """ Please create an issue on our [Github](https://github.com/ljcleo/hardcore-logic) repository to talk about your model. Then, we can test it for you and report the results here on the Leaderboard. If you would like to do local testing, please read our code [here](https://github.com/ljcleo/hardcore-logic/tree/master/src/evaluator) and apply for the access for the [HardcoreLogic](https://hf.co/dataset/?/?) that contains the truth solutions. """ gr.Markdown("## 🚀 Evaluate your models\n\n" + markdown_text, elem_classes="markdown-text") def build_demo(): global original_df, available_models, gpt4t_dfs, haiku_dfs, llama_dfs with gr.Blocks(theme=gr.themes.Soft(), css=css, js=js_light) as demo: gr.HTML(BANNER, elem_id="banner") # convert LAST_UPDATED to the PDT time LAST_UPDATED = datetime.now(pytz.timezone('US/Pacific')).strftime("%Y-%m-%d %H:%M:%S") header_md_text = HEADER_MD.replace("{LAST_UPDATED}", str(LAST_UPDATED)) gr.Markdown(header_md_text, elem_classes="markdown-text") with gr.Tabs(elem_classes="tab-buttons") as tabs: # 🏅 Leaderboard with gr.TabItem("🏅 Leaderboard", elem_id="od-benchmark-tab-table", id=0): _tab_leaderboard() # 🎯 Accuracy for each puzzl with gr.TabItem("🎯 Accuracy for each puzzle",elem_id="od-benchmark-tab-table", id=1): _tab_leaderboard_puzzle() ''' # 🚀 Evaluate your models with gr.TabItem("🚀 Evaluate your models", elem_id="od-benchmark-tab-table", id=3): _tab_submit() ''' ''' # 📮 About Us with gr.TabItem("📮 About Us", elem_id="od-benchmark-tab-table", id=4): gr.Markdown(ABOUT_MD, elem_classes="markdown-text") ''' # 📚 Citation 区域 with gr.Accordion("📚 Citation", open=False): gr.Textbox( value=CITATION_TEXT, lines=7, label="Copy this BibTeX to cite us", elem_id="citation-button", show_copy_button=True ) return demo def data_load(result_file,puzzle_file): global raw_data, original_df, raw_puzzle_data, puzzle_df print(f"Loading {result_file}") column_names_main = column_names.copy() column_puzzle_main = column_names_puzzle.copy() main_ordered_columns = ORDERED_COLUMN_NAMES puzzle_main_ordered_columns =ORDERED_COLUMN_NAMES_PUZZLE click_url = True with open(result_file, "r") as f: raw_data = json.load(f) for d in raw_data: for k, v in d.items(): try: d[k] = float(v) except: pass with open(puzzle_file, "r") as f: raw_puzzle_data = json.load(f) for d in raw_puzzle_data: for k, v in d.items(): try: d[k] = float(v) except: pass original_df = pd.DataFrame(raw_data) original_df = post_processing(original_df, column_names_main, ordered_columns=main_ordered_columns, click_url=click_url, rank_column=RANKING_COLUMN) puzzle_df = pd.DataFrame(raw_puzzle_data) puzzle_df = post_processing(puzzle_df, column_puzzle_main, ordered_columns=puzzle_main_ordered_columns, click_url=click_url, rank_column=RANKING_COLUMN) print(f"original_df.columns: {original_df.columns}") print(f"puzzle_df.columns: {puzzle_df.columns}") if __name__ == "__main__": parser = argparse.ArgumentParser() parser.add_argument("--share", action="store_true") parser.add_argument("--result_file", help="Path to results table", default="hardcorelogic.summary.json") parser.add_argument("--puzzle_file", help="Path to results(puzzle) table", default="hardcorelogic.puzzle.json") args = parser.parse_args() data_load(args.result_file,args.puzzle_file) print(original_df) demo = build_demo() demo.launch(share=args.share, height=3000, width="100%")