Spaces:
Running
Running
| import gradio as gr | |
| import pandas as pd | |
| import os | |
| from huggingface_hub import snapshot_download, login | |
| from apscheduler.schedulers.background import BackgroundScheduler | |
| from src.display.about import ( | |
| CITATION_BUTTON_LABEL, | |
| CITATION_BUTTON_TEXT, | |
| CONTACT_TEXT, | |
| EVALUATION_QUEUE_TEXT, | |
| INTRODUCTION_TEXT, | |
| LLM_BENCHMARKS_TEXT, | |
| TITLE, | |
| SUB_TITLE, | |
| ) | |
| from src.display.css_html_js import custom_css | |
| from src.envs import API | |
| from src.leaderboard.load_results import load_data | |
| # clone / pull the lmeh eval data | |
| TOKEN = os.environ.get("TOKEN", None) | |
| login(token=TOKEN) | |
| RESULTS_REPO = f"SeaLLMs/SeaExam-results" | |
| CACHE_PATH=os.getenv("HF_HOME", ".") | |
| EVAL_RESULTS_PATH = os.path.join(CACHE_PATH, "eval-results") | |
| print(EVAL_RESULTS_PATH) | |
| snapshot_download( | |
| repo_id=RESULTS_REPO, local_dir=EVAL_RESULTS_PATH, repo_type="dataset", | |
| token=TOKEN | |
| ) | |
| def restart_space(): | |
| API.restart_space(repo_id="SeaLLMs/SeaExam_leaderboard", token=TOKEN) | |
| all_columns = ['R','type', 'Model','open?', 'avg_sea ⬇️', 'en', 'zh', 'id', 'th', 'vi', 'avg', 'params(B)'] | |
| show_columns = ['R', 'Model','type','open?','params(B)', 'avg_sea ⬇️', 'en', 'zh', 'id', 'th', 'vi', 'avg', ] | |
| TYPES = ['number', 'markdown', 'str', 'str', 'number', 'number', 'number', 'number', 'number', 'number', 'number', 'number'] | |
| # Load the data from the csv file | |
| csv_path = f'{EVAL_RESULTS_PATH}/SeaExam_results_20240425.csv' | |
| df_m3exam, df_mmlu, df_avg = load_data(csv_path) | |
| # df_m3exam = df_m3exam.copy()[show_columns] | |
| # df_mmlu = df_mmlu.copy()[show_columns] | |
| df_avg_init = df_avg.copy()[df_avg['type'] == '🔶 chat'][show_columns] | |
| df_m3exam_init = df_m3exam.copy()[df_m3exam['type'] == '🔶 chat'][show_columns] | |
| df_mmlu_init = df_mmlu.copy()[df_mmlu['type'] == '🔶 chat'][show_columns] | |
| # data_types = ['number', 'str', 'markdown','str', 'number', 'number', 'number', 'number', 'number', 'number', 'number', 'number'] | |
| # map_columns = {'rank':'R','type':'type', 'Model':'Model','open?':'open?', 'avg_sea':'avg_sea ⬇️', 'en':'en', 'zh':'zh', 'id':'id', 'th':'th', 'vi':'vi', 'avg':'avg', 'params':'params(B)'} | |
| # map_types = {'rank': 'number', 'type': 'str', 'Model': 'markdown', 'open?': 'str', 'avg_sea': 'number', 'en': 'number', 'zh': 'number', 'id': 'number', 'th': 'number', 'vi': 'number', 'avg': 'number', 'params': 'number'} | |
| # Searching and filtering | |
| def update_table( | |
| hidden_df: pd.DataFrame, | |
| # columns: list, | |
| type_query: list, | |
| open_query: list, | |
| # precision_query: str, | |
| # size_query: list, | |
| # show_deleted: bool, | |
| query: str, | |
| ): | |
| # filtered_df = filter_models(hidden_df, type_query, size_query, precision_query, show_deleted) | |
| # filtered_df = filter_queries(query, filtered_df) | |
| # df = select_columns(filtered_df, columns) | |
| filtered_df = hidden_df.copy() | |
| filtered_df = filtered_df[filtered_df['type'].isin(type_query)] | |
| map_open = {'open': 'Y', 'closed': 'N'} | |
| filtered_df = filtered_df[filtered_df['open?'].isin([map_open[o] for o in open_query])] | |
| filtered_df = filter_queries(query, filtered_df) | |
| # filtered_df = filtered_df[[map_columns[k] for k in columns]] | |
| # deduplication | |
| # df = df.drop_duplicates(subset=["Model"]) | |
| df = filtered_df.drop_duplicates() | |
| df = df[show_columns] | |
| return df | |
| def search_table(df: pd.DataFrame, query: str) -> pd.DataFrame: | |
| return df[(df['Model'].str.contains(query, case=False))] | |
| def filter_queries(query: str, filtered_df: pd.DataFrame) -> pd.DataFrame: | |
| final_df = [] | |
| if query != "": | |
| queries = [q.strip() for q in query.split(";")] | |
| for _q in queries: | |
| _q = _q.strip() | |
| if _q != "": | |
| temp_filtered_df = search_table(filtered_df, _q) | |
| if len(temp_filtered_df) > 0: | |
| final_df.append(temp_filtered_df) | |
| if len(final_df) > 0: | |
| filtered_df = pd.concat(final_df) | |
| return filtered_df | |
| demo = gr.Blocks(css=custom_css) | |
| with demo: | |
| gr.HTML(TITLE) | |
| gr.HTML(SUB_TITLE) | |
| gr.Markdown(INTRODUCTION_TEXT, elem_classes="markdown-text") | |
| with gr.Tabs(elem_classes="tab-buttons") as tabs: | |
| with gr.TabItem("🏅 Overall", elem_id="llm-benchmark-Sum", id=0): | |
| with gr.Row(): | |
| with gr.Column(): | |
| with gr.Row(): | |
| search_bar = gr.Textbox( | |
| placeholder=" 🔍 Search for your model (separate multiple queries with `;`) and press ENTER...", | |
| show_label=False, | |
| elem_id="search-bar", | |
| ) | |
| # with gr.Row(): | |
| # with gr.Column(): | |
| # shown_columns = gr.CheckboxGroup( | |
| # choices=["rank","type", "Model","open?", "avg_sea", "en", "zh", "id", "th", "vi", "avg", "params"], | |
| # value=["rank", "type", "Model", "avg_sea", "en", "zh", "id", "th", "vi", "avg", "params"], | |
| # label="Select model types to show", | |
| # elem_id="column-select", | |
| # interactive=True, | |
| # ) | |
| # with gr.Row(): | |
| with gr.Column(): | |
| type_query = gr.CheckboxGroup( | |
| choices=["🟢 base", "🔶 chat"], | |
| value=["🔶 chat" ], | |
| label="model types to show", | |
| elem_id="type-select", | |
| interactive=True, | |
| ) | |
| with gr.Column(): | |
| open_query = gr.CheckboxGroup( | |
| choices=["open", "closed"], | |
| value=["open", "closed"], | |
| label="open-source or closed-source models?", | |
| elem_id="open-select", | |
| interactive=True, | |
| ) | |
| leaderboard_table = gr.components.Dataframe( | |
| value=df_avg_init, | |
| # [[map_columns[k] for k in shown_columns.value]], | |
| # value=leaderboard_df[ | |
| # [c.name for c in fields(AutoEvalColumn) if c.never_hidden] | |
| # + shown_columns.value | |
| # + [AutoEvalColumn.dummy.name] | |
| # ], | |
| # headers=[c.name for c in fields(AutoEvalColumn) if c.never_hidden] + shown_columns.value, | |
| datatype=TYPES, | |
| elem_id="leaderboard-table", | |
| interactive=False, | |
| # datatype=['number', 'str', 'markdown', 'number', 'number', 'number', 'number', 'number', 'number', 'number', 'number'], | |
| # datatype=[map_types[k] for k in shown_columns.value], | |
| visible=True, | |
| # column_widths=["20%", "6%", "8%", "6%", "8%", "8%", "6%", "6%", "6%", "6%", "6%"], | |
| ) | |
| hidden_leaderboard_table_for_search = gr.components.Dataframe( | |
| value=df_avg, | |
| # elem_id="leaderboard-table", | |
| interactive=False, | |
| visible=False, | |
| ) | |
| search_bar.submit( | |
| update_table, | |
| [ | |
| # df_avg, | |
| hidden_leaderboard_table_for_search, | |
| # shown_columns, | |
| type_query, | |
| open_query, | |
| # filter_columns_type, | |
| # filter_columns_precision, | |
| # filter_columns_size, | |
| # deleted_models_visibility, | |
| search_bar, | |
| ], | |
| leaderboard_table, | |
| ) | |
| for selector in [type_query, open_query]: | |
| selector.change( | |
| update_table, | |
| [ | |
| # df_avg, | |
| hidden_leaderboard_table_for_search, | |
| # shown_columns, | |
| type_query, | |
| open_query, | |
| # filter_columns_type, | |
| # filter_columns_precision, | |
| # filter_columns_size, | |
| # deleted_models_visibility, | |
| search_bar, | |
| ], | |
| leaderboard_table, | |
| ) | |
| with gr.TabItem("M3Exam", elem_id="llm-benchmark-M3Exam", id=1): | |
| with gr.Row(): | |
| with gr.Column(): | |
| search_bar = gr.Textbox( | |
| placeholder=" 🔍 Search for your model (separate multiple queries with `;`) and press ENTER...", | |
| show_label=False, | |
| elem_id="search-bar", | |
| ) | |
| with gr.Column(): | |
| type_query = gr.CheckboxGroup( | |
| choices=["🟢 base", "🔶 chat"], | |
| value=["🔶 chat" ], | |
| label="model types to show", | |
| elem_id="type-select", | |
| interactive=True, | |
| ) | |
| with gr.Column(): | |
| open_query = gr.CheckboxGroup( | |
| choices=["open", "closed"], | |
| value=["open", "closed"], | |
| label="open-source or closed-source models?", | |
| elem_id="open-select", | |
| interactive=True, | |
| ) | |
| leaderboard_table = gr.components.Dataframe( | |
| value=df_m3exam_init, | |
| interactive=False, | |
| visible=True, | |
| # datatype=['number', 'str', 'markdown', 'number', 'number', 'number', 'number', 'number', 'number', 'number', 'number'], | |
| datatype=TYPES, | |
| ) | |
| hidden_leaderboard_table_for_search = gr.components.Dataframe( | |
| value=df_m3exam, | |
| interactive=False, | |
| visible=False, | |
| ) | |
| search_bar.submit( | |
| update_table, | |
| [ | |
| hidden_leaderboard_table_for_search, | |
| type_query, | |
| open_query, | |
| search_bar, | |
| ], | |
| leaderboard_table, | |
| ) | |
| for selector in [type_query, open_query]: | |
| selector.change( | |
| update_table, | |
| [ | |
| hidden_leaderboard_table_for_search, | |
| type_query, | |
| open_query, | |
| search_bar, | |
| ], | |
| leaderboard_table, | |
| ) | |
| with gr.TabItem("MMLU", elem_id="llm-benchmark-MMLU", id=2): | |
| with gr.Row(): | |
| with gr.Column(): | |
| search_bar = gr.Textbox( | |
| placeholder=" 🔍 Search for your model (separate multiple queries with `;`) and press ENTER...", | |
| show_label=False, | |
| elem_id="search-bar", | |
| ) | |
| with gr.Column(): | |
| type_query = gr.CheckboxGroup( | |
| choices=["🟢 base", "🔶 chat"], | |
| value=["🔶 chat" ], | |
| label="model types to show", | |
| elem_id="type-select", | |
| interactive=True, | |
| ) | |
| with gr.Column(): | |
| open_query = gr.CheckboxGroup( | |
| choices=["open", "closed"], | |
| value=["open", "closed"], | |
| label="open-source or closed-source models?", | |
| elem_id="open-select", | |
| interactive=True, | |
| ) | |
| leaderboard_table = gr.components.Dataframe( | |
| value=df_mmlu_init, | |
| interactive=False, | |
| visible=True, | |
| # datatype=['number', 'str', 'markdown', 'number', 'number', 'number', 'number', 'number', 'number', 'number', 'number'], | |
| datatype=TYPES, | |
| ) | |
| hidden_leaderboard_table_for_search = gr.components.Dataframe( | |
| value=df_mmlu, | |
| interactive=False, | |
| visible=False, | |
| ) | |
| search_bar.submit( | |
| update_table, | |
| [ | |
| hidden_leaderboard_table_for_search, | |
| type_query, | |
| open_query, | |
| search_bar, | |
| ], | |
| leaderboard_table, | |
| ) | |
| for selector in [type_query, open_query]: | |
| selector.change( | |
| update_table, | |
| [ | |
| hidden_leaderboard_table_for_search, | |
| type_query, | |
| open_query, | |
| search_bar, | |
| ], | |
| leaderboard_table, | |
| ) | |
| with gr.TabItem("📝 About", elem_id="llm-benchmark-tab-table", id=3): | |
| gr.Markdown(LLM_BENCHMARKS_TEXT, elem_classes="markdown-text") | |
| # with gr.Row(): | |
| # with gr.Accordion("📙 Citation", open=False): | |
| # citation_button = gr.Textbox( | |
| # value=CITATION_BUTTON_TEXT, | |
| # label=CITATION_BUTTON_LABEL, | |
| # lines=20, | |
| # elem_id="citation-button", | |
| # show_copy_button=True, | |
| # ) | |
| gr.Markdown(CONTACT_TEXT, elem_classes="markdown-text") | |
| demo.launch() | |
| scheduler = BackgroundScheduler() | |
| scheduler.add_job(restart_space, "interval", seconds=1800) | |
| scheduler.start() | |
| demo.queue(default_concurrency_limit=40).launch(share=True) | |