Upload folder using huggingface_hub

app/draw_diagram.py

@@ -20,8 +20,6 @@ from model_information import get_dataframe
 
 info_df = get_dataframe()
 
-
-
 # def nav_to(value):
 #     try:
 #         url = links_dic[str(value).lower()]
@@ -90,6 +88,16 @@ def draw(folder_name, category_name, dataset_name, metrics, cus_sort=True):
 
         cur_dataset_name = chart_data_table.columns[1]
 
+
+        def highlight_first_element(x):
+                # Create a DataFrame with the same shape as the input
+                df_style = pd.DataFrame('', index=x.index, columns=x.columns)
+                
+                # Apply background color to the first element in row 0 (df[0][0])
+                df_style.iloc[0, 1] = 'background-color: #b0c1d7; color: white'
+                
+                return df_style
+
         if cur_dataset_name in [
                             'librispeech_test_clean',
                             'librispeech_test_other',
@@ -105,19 +113,35 @@ def draw(folder_name, category_name, dataset_name, metrics, cus_sort=True):
                             'aishell_asr_zh_test',
                             ]:
             
-            styled_df = chart_data_table.style.highlight_min(
-                subset=[chart_data_table.columns[1]], color='yellow'
-            )
+            chart_data_table = chart_data_table.sort_values(
+                    by=chart_data_table.columns[1],
+                    ascending=True
+                ).reset_index(drop=True)
         else:
-
             chart_data_table = chart_data_table.sort_values(
                     by=chart_data_table.columns[1],
                     ascending=False
                 ).reset_index(drop=True)
+            
+            # styled_df = chart_data_table.style.highlight_min(
+            #     subset=[chart_data_table.columns[1]], color='yellow'
+            # )
 
-            styled_df = chart_data_table.style.highlight_max(
-                subset=[chart_data_table.columns[1]], color='yellow'
-            )
+        styled_df = chart_data_table.style.apply(
+            highlight_first_element, axis=None
+        )
+
+        # else:
+
+            
+
+        #     # styled_df = chart_data_table.style.highlight_max(
+        #     #     subset=[chart_data_table.columns[1]], color='yellow'
+        #     # )
+
+        #     styled_df = chart_data_table.style.apply(
+        #         highlight_first_element, axis=None
+        #     )
 
 
         st.dataframe(

app/pages.py

@@ -1,6 +1,7 @@
 import streamlit as st
 from app.draw_diagram import *
 from app.content import *
+from app.summarization import *
 
 def dataset_contents(dataset, metrics):
     
@@ -87,18 +88,22 @@ def dashboard():
 def asr():
     st.title("Task: Automatic Speech Recognition")
     
-    filters_levelone = ['LibriSpeech-Test-Clean', 
-                        'LibriSpeech-Test-Other', 
-                        'Common-Voice-15-En-Test', 
-                        'Peoples-Speech-Test', 
-                        'GigaSpeech-Test', 
-                        'Earnings21-Test', 
-                        'Earnings22-Test', 
-                        'Tedlium3-Test', 
-                        'Tedlium3-Long-form-Test', 
-                        #'IMDA-Part1-ASR-Test', 
-                        #'IMDA-Part2-ASR-Test'
-                        ]
+    sum = ['Summarization']
+
+    dataset_lists = ['LibriSpeech-Test-Clean', 
+                    'LibriSpeech-Test-Other', 
+                    'Common-Voice-15-En-Test', 
+                    'Peoples-Speech-Test', 
+                    'GigaSpeech-Test', 
+                    'Earnings21-Test', 
+                    'Earnings22-Test', 
+                    'Tedlium3-Test', 
+                    'Tedlium3-Long-form-Test', 
+                    #'IMDA-Part1-ASR-Test', 
+                    #'IMDA-Part2-ASR-Test'
+                    ]
+
+    filters_levelone = sum + dataset_lists
     
     left, center, _, middle,right = st.columns([0.2, 0.2, 0.2, 0.2 ,0.2])
     
@@ -106,21 +111,26 @@ def asr():
         filter_1 = st.selectbox('Dataset', filters_levelone)
     
     if filter_1:
-        dataset_contents(asr_datsets[filter_1], metrics['wer'])
-        draw('su', 'ASR', filter_1, 'wer', cus_sort=True)
+        if filter_1 in sum:
+            sum_table_mulit_metrix('ASR', ['wer'])
+        else:
+            dataset_contents(asr_datsets[filter_1], metrics['wer'])
+            draw('su', 'ASR', filter_1, 'wer', cus_sort=True)
 
     
 
 def sqa():
     st.title("Task: Speech Question Answering")
     
+    sum = ['Summarization']
+
     binary = ['CN-College-Listen-MCQ-Test', 'DREAM-TTS-MCQ-Test']
 
     rest = ['SLUE-P2-SQA5-Test', 
             'Public-SG-Speech-QA-Test', 
             'Spoken-Squad-Test']
 
-    filters_levelone = binary + rest
+    filters_levelone = sum + binary + rest
     
     left, center, _, middle,right = st.columns([0.2, 0.2, 0.2, 0.2 ,0.2])
     
@@ -128,7 +138,10 @@ def sqa():
         filter_1 = st.selectbox('Dataset', filters_levelone)
 
     if filter_1:
-        if filter_1 in binary:
+        if filter_1 in sum:
+            sum_table_mulit_metrix('SQA', ['llama3_70b_judge_binary', 'llama3_70b_judge'])
+
+        elif filter_1 in binary:
             dataset_contents(sqa_datasets[filter_1], metrics['llama3_70b_judge_binary'])
             draw('su', 'SQA', filter_1, 'llama3_70b_judge_binary')
         
@@ -139,8 +152,12 @@ def sqa():
 def si():
     st.title("Task: Speech Instruction")
     
-    filters_levelone = ['OpenHermes-Audio-Test', 
-                        'ALPACA-Audio-Test']
+    sum = ['Summarization']
+
+    dataset_lists = ['OpenHermes-Audio-Test', 
+                     'ALPACA-Audio-Test']
+    
+    filters_levelone = sum + dataset_lists
     
     left, center, _, middle,right = st.columns([0.2, 0.2, 0.2, 0.2 ,0.2])
     
@@ -148,8 +165,11 @@ def si():
         filter_1 = st.selectbox('Dataset', filters_levelone)
 
     if filter_1:
-        dataset_contents(si_datasets[filter_1], metrics['llama3_70b_judge'])
-        draw('su', 'SI', filter_1, 'llama3_70b_judge')
+        if filter_1 in sum:
+            sum_table_mulit_metrix('SI', ['llama3_70b_judge'])
+        else:
+            dataset_contents(si_datasets[filter_1], metrics['llama3_70b_judge'])
+            draw('su', 'SI', filter_1, 'llama3_70b_judge')
 
 def ac():
     st.title("Task: Audio Captioning")
@@ -173,9 +193,13 @@ def ac():
 def asqa():
     st.title("Task: Audio Scene Question Answering")
 
-    filters_levelone = ['Clotho-AQA-Test', 
-                        'WavCaps-QA-Test', 
-                        'AudioCaps-QA-Test']
+    sum = ['Summarization']
+
+    dataset_lists = ['Clotho-AQA-Test', 
+                    'WavCaps-QA-Test', 
+                    'AudioCaps-QA-Test']
+    
+    filters_levelone = sum + dataset_lists
     
     left, center, _, middle,right = st.columns([0.2, 0.2, 0.2, 0.2 ,0.2])
     
@@ -183,16 +207,23 @@ def asqa():
         filter_1 = st.selectbox('Dataset', filters_levelone)
     
     if filter_1:
-        dataset_contents(asqa_datasets[filter_1], metrics['llama3_70b_judge'])
-        draw('asu', 'AQA',filter_1, 'llama3_70b_judge')
+        if filter_1 in sum:
+            sum_table_mulit_metrix('AQA', ['llama3_70b_judge'])
+        else:
+            dataset_contents(asqa_datasets[filter_1], metrics['llama3_70b_judge'])
+            draw('asu', 'AQA',filter_1, 'llama3_70b_judge')
 
 
 def er():
     st.title("Task: Emotion Recognition")
 
-    filters_levelone = ['IEMOCAP-Emotion-Test', 
+    sum = ['Summarization']
+
+    dataset_lists = ['IEMOCAP-Emotion-Test', 
                         'MELD-Sentiment-Test', 
                         'MELD-Emotion-Test']
+
+    filters_levelone = sum + dataset_lists
     
     left, center, _, middle,right = st.columns([0.2, 0.2, 0.2, 0.2 ,0.2])
     
@@ -200,8 +231,11 @@ def er():
         filter_1 = st.selectbox('Dataset', filters_levelone)
 
     if filter_1:
-        dataset_contents(er_datasets[filter_1], metrics['llama3_70b_judge_binary'])
-        draw('vu', 'ER', filter_1, 'llama3_70b_judge_binary')
+        if filter_1 in sum:
+            sum_table_mulit_metrix('ER', ['llama3_70b_judge_binary'])
+        else:
+            dataset_contents(er_datasets[filter_1], metrics['llama3_70b_judge_binary'])
+            draw('vu', 'ER', filter_1, 'llama3_70b_judge_binary')
 
 
 def ar():
@@ -216,15 +250,21 @@ def ar():
 
 
     if filter_1:
+        # if filter_1 in sum:
+        #     sum_table('aR', 'llama3_70b_judge')
+        # else:
         dataset_contents(ar_datsets[filter_1], metrics['llama3_70b_judge'])
         draw('vu', 'AR', filter_1, 'llama3_70b_judge')
 
 
 def gr():
     st.title("Task: Gender Recognition")
+    sum = ['Summarization']
 
-    filters_levelone = ['VoxCeleb-Gender-Test', 
+    dataset_lists =  ['VoxCeleb-Gender-Test', 
                         'IEMOCAP-Gender-Test']
+
+    filters_levelone = sum + dataset_lists
     
     left, center, _, middle,right = st.columns([0.2, 0.2, 0.2, 0.2 ,0.2])
     
@@ -232,19 +272,24 @@ def gr():
         filter_1 = st.selectbox('Dataset', filters_levelone)
     
     if filter_1:
-        dataset_contents(gr_datasets[filter_1], metrics['llama3_70b_judge_binary'])
-        draw('vu', 'GR', filter_1, 'llama3_70b_judge_binary')
+        if filter_1 in sum:
+            sum_table_mulit_metrix('GR', ['llama3_70b_judge_binary'])
+        else:
+            dataset_contents(gr_datasets[filter_1], metrics['llama3_70b_judge_binary'])
+            draw('vu', 'GR', filter_1, 'llama3_70b_judge_binary')
 
 
 def spt():
     st.title("Task: Speech Translation")
-
-    filters_levelone = ['Covost2-EN-ID-test', 
+    sum = ['Summarization']
+    dataset_lists = ['Covost2-EN-ID-test', 
                         'Covost2-EN-ZH-test',
                         'Covost2-EN-TA-test', 
                         'Covost2-ID-EN-test', 
                         'Covost2-ZH-EN-test', 
                         'Covost2-TA-EN-test']
+
+    filters_levelone = sum + dataset_lists
     
     left, center, _, middle,right = st.columns([0.2, 0.2, 0.2, 0.2 ,0.2])
     
@@ -252,8 +297,11 @@ def spt():
         filter_1 = st.selectbox('Dataset', filters_levelone)
     
     if filter_1:
-        dataset_contents(spt_datasets[filter_1], metrics['bleu'])
-        draw('su', 'ST', filter_1, 'bleu')
+        if filter_1 in sum:
+            sum_table_mulit_metrix('ST', ['bleu'])
+        else:
+            dataset_contents(spt_datasets[filter_1], metrics['bleu'])
+            draw('su', 'ST', filter_1, 'bleu')
 
 
 def cnasr():

app/summarization.py

@@ -0,0 +1,118 @@
+import streamlit as st
+import pandas as pd
+import numpy as np
+from streamlit_echarts import st_echarts
+from streamlit.components.v1 import html
+# from PIL import Image 
+from app.show_examples import *
+import pandas as pd
+from typing import List
+
+from model_information import get_dataframe
+
+info_df = get_dataframe()
+
+metrics_info = {
+    'wer': 'Word Error Rate (WER), a common metric for ASR evaluation. (The lower, the better)',
+    'llama3_70b_judge_binary': 'Binary evaluation using the LLAMA3-70B model, for tasks requiring a binary outcome. (0-100 based on score 0-1)',
+    'llama3_70b_judge': 'General evaluation using the LLAMA3-70B model, typically scoring based on subjective judgments. (0-100 based on score 0-5)',
+    'meteor': 'METEOR, a metric used for evaluating text generation, often used in translation or summarization tasks. (Sensitive to output length)',
+    'bleu': 'BLEU (Bilingual Evaluation Understudy), another text generation evaluation metric commonly used in machine translation. (Sensitive to output length)',
+}
+
+def sum_table_mulit_metrix(task_name, metrics_lists: List[str]):
+    
+    for metrics in metrics_lists:
+        folder = f"./results/{metrics}/"
+        data_path = f'{folder}/{task_name.lower()}.csv'
+
+        chart_data = pd.read_csv(data_path).round(3)
+        selected_columns = [i for i in chart_data.columns if i != 'Model']
+        chart_data['Average'] = chart_data[selected_columns].mean(axis=1)
+        
+        # new_dataset_name = dataset_name.replace('-', '_').lower()
+        
+        st.markdown("""
+                    <style>
+                    .stMultiSelect [data-baseweb=select] span {
+                        max-width: 800px;
+                        font-size: 0.9rem;
+                        background-color: #3C6478 !important; /* Background color for selected items */
+                        color: white; /* Change text color */
+                        back
+                    }
+                    </style>
+                    """, unsafe_allow_html=True)
+        
+        # remap model names
+        display_model_names = {key.strip() :val.strip() for key, val in zip(info_df['Original Name'], info_df['Proper Display Name'])}
+        chart_data['model_show'] = chart_data['Model'].map(lambda x: display_model_names.get(x, x))
+
+        models = st.multiselect("Please choose the model", 
+                                sorted(chart_data['model_show'].tolist()), 
+                                default = sorted(chart_data['model_show'].tolist()),
+                                key=f"multiselect_{task_name}_{metrics}"
+                                )
+        
+        chart_data = chart_data[chart_data['model_show'].isin(models)].dropna(axis=0)
+        # chart_data = chart_data.sort_values(by=['Average'], ascending=True).dropna(axis=0)
+
+        if len(chart_data) == 0: return
+
+        # = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = =
+        '''
+        Show Table
+        '''
+        with st.container():
+            st.markdown(f'#### Overal Evaluation Results')
+            st.markdown(f'###### Evaluation Method: {metrics_info[metrics]}')
+
+            model_link = {key.strip(): val for key, val in zip(info_df['Proper Display Name'], info_df['Link'])}
+
+            chart_data['model_link'] = chart_data['model_show'].map(model_link) 
+
+            tabel_columns = [i for i in chart_data.columns if i not in ['Model', 'model_show']]
+            column_to_front = 'Average'
+            new_order = [column_to_front] + [col for col in tabel_columns if col != column_to_front]
+            
+            chart_data_table = chart_data[['model_show'] + new_order]
+            
+
+            # Format numeric columns to 2 decimal places
+            chart_data_table[chart_data_table.columns[1]] = chart_data_table[chart_data_table.columns[1]].apply(lambda x: round(float(x), 3) if isinstance(float(x), (int, float)) else float(x))
+
+            if metrics in ['wer']:
+                ascend = True
+            else:
+                ascend= False
+
+            chart_data_table = chart_data_table.sort_values(
+                    by=['Average'],
+                    ascending=ascend
+                ).reset_index(drop=True)
+            
+            def highlight_first_element(x):
+                # Create a DataFrame with the same shape as the input
+                df_style = pd.DataFrame('', index=x.index, columns=x.columns)
+                
+                # Apply background color to the first element in row 0 (df[0][0])
+                df_style.iloc[0, 1] = 'background-color: #b0c1d7; color: white'
+                
+                return df_style
+            
+            styled_df = chart_data_table.style.apply(
+                highlight_first_element, axis=None
+            )
+
+            st.dataframe(
+                    styled_df,
+                    column_config={
+                        'model_show': 'Model',
+                        chart_data_table.columns[1]: {'alignment': 'left'},
+                        "model_link": st.column_config.LinkColumn(
+                            "Model Link",
+                        ),
+                    },
+                    hide_index=True,
+                    use_container_width=True
+                )