Upload classification_report_llama3.2_3b_oos_full.txt

predictions-zeroshot/round8-fewshot-1exampleeach-k-knownintent-restoos-100oossentences/classification_report_llama3.2_3b_oos_full.txt

@@ -0,0 +1,706 @@
+
+================================================================================
+
+
+
+Run: 1notoos
+Overall Accuracy: 100.00%
+Overall F1: 100.00%
+              precision    recall  f1-score   support
+
+         oos       1.00      1.00      1.00       100
+
+    accuracy                           1.00       100
+   macro avg       1.00      1.00      1.00       100
+weighted avg       1.00      1.00      1.00       100
+
+predicted
+oos    100
+Name: count, dtype: int64
+
+Classification report saved to classification_report_llama3.2:3b_oos1notoos.txt
+
+
+================================================================================
+
+
+Run: 2notoos
+Overall Accuracy: 80.00%
+Overall F1: 88.89%
+                   precision    recall  f1-score   support
+
+              oos       1.00      0.80      0.89       100
+travel_suggestion       0.00       nan      0.00         0
+    change_volume       0.00       nan      0.00         0
+
+         accuracy                           0.80       100
+        macro avg       0.33      0.80      0.30       100
+     weighted avg       1.00      0.80      0.89       100
+
+predicted
+oos                  80
+change_volume        15
+travel_suggestion     5
+Name: count, dtype: int64
+
+Classification report saved to classification_report_llama3.2:3b_oos2notoos.txt
+
+
+
+
+
+================================================================================
+
+
+Run: 3notoos
+Overall Accuracy: 90.00%
+Overall F1: 94.74%
+                   precision    recall  f1-score   support
+
+              oos       1.00      0.90      0.95       100
+    change_volume       0.00       nan      0.00         0
+travel_suggestion       0.00       nan      0.00         0
+
+         accuracy                           0.90       100
+        macro avg       0.33      0.90      0.32       100
+     weighted avg       1.00      0.90      0.95       100
+
+predicted
+oos                  90
+change_volume         7
+travel_suggestion     3
+Name: count, dtype: int64
+
+Classification report saved to classification_report_llama3.2:3b_oos3notoos.txt
+
+
+
+
+
+
+================================================================================
+
+Run: 4notoos
+Overall Accuracy: 78.00%
+Overall F1: 87.64%
+                   precision    recall  f1-score   support
+
+              oos       1.00      0.78      0.88       100
+    change_volume       0.00       nan      0.00         0
+           cancel       0.00       nan      0.00         0
+travel_suggestion       0.00       nan      0.00         0
+
+         accuracy                           0.78       100
+        macro avg       0.25      0.78      0.22       100
+     weighted avg       1.00      0.78      0.88       100
+
+predicted
+oos                  78
+change_volume        11
+cancel                8
+travel_suggestion     3
+Name: count, dtype: int64
+
+Classification report saved to classification_report_llama3.2:3b_oos4notoos.txt
+
+
+
+
+
+
+
+================================================================================
+
+
+Run: 5notoos
+Overall Accuracy: 75.00%
+Overall F1: 85.71%
+                     precision    recall  f1-score   support
+
+                oos       1.00      0.75      0.86       100
+  travel_suggestion       0.00       nan      0.00         0
+travel_notification       0.00       nan      0.00         0
+      change_volume       0.00       nan      0.00         0
+           pto_used       0.00       nan      0.00         0
+             cancel       0.00       nan      0.00         0
+
+           accuracy                           0.75       100
+          macro avg       0.17      0.75      0.14       100
+       weighted avg       1.00      0.75      0.86       100
+
+predicted
+oos                    75
+travel_suggestion       9
+change_volume           8
+pto_used                3
+cancel                  3
+travel_notification     2
+Name: count, dtype: int64
+
+Classification report saved to classification_report_llama3.2:3b_oos5notoos.txt
+
+
+
+
+================================================================================
+
+
+
+Run: 10notoos
+Overall Accuracy: 69.00%
+Overall F1: 81.66%
+                       precision    recall  f1-score   support
+
+                  oos       1.00      0.69      0.82       100
+    travel_suggestion       0.00       nan      0.00         0
+          min_payment       0.00       nan      0.00         0
+restaurant_suggestion       0.00       nan      0.00         0
+        change_volume       0.00       nan      0.00         0
+               cancel       0.00       nan      0.00         0
+  travel_notification       0.00       nan      0.00         0
+                  mpg       0.00       nan      0.00         0
+     change_user_name       0.00       nan      0.00         0
+             pto_used       0.00       nan      0.00         0
+
+             accuracy                           0.69       100
+            macro avg       0.10      0.69      0.08       100
+         weighted avg       1.00      0.69      0.82       100
+
+predicted
+oos                      69
+min_payment              11
+travel_suggestion         8
+change_volume             4
+cancel                    3
+restaurant_suggestion     1
+travel_notification       1
+mpg                       1
+change_user_name          1
+pto_used                  1
+Name: count, dtype: int64
+
+Classification report saved to classification_report_llama3.2:3b_oos10notoos.txt
+
+
+
+
+
+================================================================================
+
+
+Run: 11notoos
+Overall Accuracy: 79.00%
+Overall F1: 88.27%
+                       precision    recall  f1-score   support
+
+                  oos       1.00      0.79      0.88       100
+    travel_suggestion       0.00       nan      0.00         0
+          min_payment       0.00       nan      0.00         0
+restaurant_suggestion       0.00       nan      0.00         0
+        change_volume       0.00       nan      0.00         0
+  travel_notification       0.00       nan      0.00         0
+               cancel       0.00       nan      0.00         0
+                  mpg       0.00       nan      0.00         0
+     change_user_name       0.00       nan      0.00         0
+             pto_used       0.00       nan      0.00         0
+
+             accuracy                           0.79       100
+            macro avg       0.10      0.79      0.09       100
+         weighted avg       1.00      0.79      0.88       100
+
+predicted
+oos                      79
+travel_suggestion         6
+min_payment               6
+restaurant_suggestion     2
+change_volume             2
+travel_notification       1
+cancel                    1
+mpg                       1
+change_user_name          1
+pto_used                  1
+Name: count, dtype: int64
+
+Classification report saved to classification_report_llama3.2:3b_oos11notoos.txt
+
+
+
+
+
+
+================================================================================
+
+Run: 12notoos
+Overall Accuracy: 45.00%
+Overall F1: 62.07%
+                     precision    recall  f1-score   support
+
+         definition       0.00       nan      0.00         0
+                oos       1.00      0.45      0.62       100
+  travel_suggestion       0.00       nan      0.00         0
+        min_payment       0.00       nan      0.00         0
+      change_volume       0.00       nan      0.00         0
+travel_notification       0.00       nan      0.00         0
+             cancel       0.00       nan      0.00         0
+                mpg       0.00       nan      0.00         0
+   change_user_name       0.00       nan      0.00         0
+       next_holiday       0.00       nan      0.00         0
+
+           accuracy                           0.45       100
+          macro avg       0.10      0.45      0.06       100
+       weighted avg       1.00      0.45      0.62       100
+
+predicted
+oos                    45
+definition             36
+min_payment             6
+travel_suggestion       5
+change_volume           3
+travel_notification     1
+cancel                  1
+mpg                     1
+change_user_name        1
+next_holiday            1
+Name: count, dtype: int64
+
+Classification report saved to classification_report_llama3.2:3b_oos12notoos.txt
+
+
+
+
+
+
+================================================================================
+
+Run: 13notoos
+Overall Accuracy: 33.00%
+Overall F1: 49.62%
+                       precision    recall  f1-score   support
+
+           definition       0.00       nan      0.00         0
+                  oos       1.00      0.33      0.50       100
+    travel_suggestion       0.00       nan      0.00         0
+restaurant_suggestion       0.00       nan      0.00         0
+               cancel       0.00       nan      0.00         0
+           calculator       0.00       nan      0.00         0
+  travel_notification       0.00       nan      0.00         0
+          min_payment       0.00       nan      0.00         0
+                  mpg       0.00       nan      0.00         0
+     change_user_name       0.00       nan      0.00         0
+        change_volume       0.00       nan      0.00         0
+         next_holiday       0.00       nan      0.00         0
+
+             accuracy                           0.33       100
+            macro avg       0.08      0.33      0.04       100
+         weighted avg       1.00      0.33      0.50       100
+
+predicted
+definition               34
+oos                      33
+travel_suggestion        15
+restaurant_suggestion     5
+cancel                    4
+min_payment               2
+mpg                       2
+calculator                1
+travel_notification       1
+change_user_name          1
+change_volume             1
+next_holiday              1
+Name: count, dtype: int64
+
+Classification report saved to classification_report_llama3.2:3b_oos13notoos.txt
+
+
+
+
+
+
+================================================================================
+
+Run: 14notoos
+Overall Accuracy: 43.00%
+Overall F1: 60.14%
+                       precision    recall  f1-score   support
+
+           definition       0.00       nan      0.00         0
+                  oos       1.00      0.43      0.60       100
+    travel_suggestion       0.00       nan      0.00         0
+   international_fees       0.00       nan      0.00         0
+restaurant_suggestion       0.00       nan      0.00         0
+               cancel       0.00       nan      0.00         0
+           calculator       0.00       nan      0.00         0
+  travel_notification       0.00       nan      0.00         0
+                  mpg       0.00       nan      0.00         0
+     change_user_name       0.00       nan      0.00         0
+
+             accuracy                           0.43       100
+            macro avg       0.10      0.43      0.06       100
+         weighted avg       1.00      0.43      0.60       100
+
+predicted
+oos                      43
+definition               31
+travel_suggestion        10
+international_fees        5
+restaurant_suggestion     5
+cancel                    2
+calculator                1
+travel_notification       1
+mpg                       1
+change_user_name          1
+Name: count, dtype: int64
+
+Classification report saved to classification_report_llama3.2:3b_oos14notoos.txt
+
+
+
+
+
+
+================================================================================
+
+
+Run: 15notoos
+Overall Accuracy: 28.00%
+Overall F1: 43.75%
+                       precision    recall  f1-score   support
+
+           definition       0.00       nan      0.00         0
+     current_location       0.00       nan      0.00         0
+restaurant_suggestion       0.00       nan      0.00         0
+    travel_suggestion       0.00       nan      0.00         0
+   international_fees       0.00       nan      0.00         0
+                  oos       1.00      0.28      0.44       100
+           calculator       0.00       nan      0.00         0
+               cancel       0.00       nan      0.00         0
+  travel_notification       0.00       nan      0.00         0
+                  mpg       0.00       nan      0.00         0
+        change_volume       0.00       nan      0.00         0
+     change_user_name       0.00       nan      0.00         0
+
+             accuracy                           0.28       100
+            macro avg       0.08      0.28      0.04       100
+         weighted avg       1.00      0.28      0.44       100
+
+predicted
+definition               41
+oos                      28
+current_location          6
+restaurant_suggestion     5
+travel_suggestion         5
+international_fees        5
+calculator                3
+cancel                    3
+travel_notification       1
+mpg                       1
+change_volume             1
+change_user_name          1
+Name: count, dtype: int64
+
+Classification report saved to classification_report_llama3.2:3b_oos15notoos.txt
+
+
+
+
+
+
+================================================================================
+
+
+
+Run: 20notoos
+Overall Accuracy: 24.00%
+Overall F1: 38.71%
+                         precision    recall  f1-score   support
+
+      travel_suggestion       0.00       nan      0.00         0
+  restaurant_suggestion       0.00       nan      0.00         0
+                    oos       1.00      0.24      0.39       100
+             calculator       0.00       nan      0.00         0
+               calories       0.00       nan      0.00         0
+             definition       0.00       nan      0.00         0
+       current_location       0.00       nan      0.00         0
+ingredient_substitution       0.00       nan      0.00         0
+   schedule_maintenance       0.00       nan      0.00         0
+                    apr       0.00       nan      0.00         0
+                 cancel       0.00       nan      0.00         0
+                    mpg       0.00       nan      0.00         0
+          change_volume       0.00       nan      0.00         0
+     international_fees       0.00       nan      0.00         0
+
+               accuracy                           0.24       100
+              macro avg       0.07      0.24      0.03       100
+           weighted avg       1.00      0.24      0.39       100
+
+predicted
+definition                 28
+oos                        24
+restaurant_suggestion      15
+travel_suggestion          12
+calculator                  6
+calories                    3
+ingredient_substitution     3
+current_location            2
+international_fees          2
+schedule_maintenance        1
+apr                         1
+cancel                      1
+mpg                         1
+change_volume               1
+Name: count, dtype: int64
+
+Classification report saved to classification_report_llama3.2:3b_oos20notoos.txt
+
+
+
+
+
+
+================================================================================
+
+
+
+Run: 30notoos
+Overall Accuracy: 30.00%
+Overall F1: 46.15%
+                         precision    recall  f1-score   support
+
+               fun_fact       0.00       nan      0.00         0
+  restaurant_suggestion       0.00       nan      0.00         0
+      travel_suggestion       0.00       nan      0.00         0
+             calculator       0.00       nan      0.00         0
+               calories       0.00       nan      0.00         0
+             definition       0.00       nan      0.00         0
+                    oos       1.00      0.30      0.46       100
+       current_location       0.00       nan      0.00         0
+   schedule_maintenance       0.00       nan      0.00         0
+           travel_alert       0.00       nan      0.00         0
+     international_fees       0.00       nan      0.00         0
+                  maybe       0.00       nan      0.00         0
+                    mpg       0.00       nan      0.00         0
+            sync_device       0.00       nan      0.00         0
+ingredient_substitution       0.00       nan      0.00         0
+
+               accuracy                           0.30       100
+              macro avg       0.07      0.30      0.03       100
+           weighted avg       1.00      0.30      0.46       100
+
+predicted
+oos                        30
+definition                 22
+travel_suggestion          13
+restaurant_suggestion       9
+fun_fact                    6
+calculator                  4
+international_fees          4
+calories                    3
+travel_alert                2
+maybe                       2
+schedule_maintenance        1
+current_location            1
+mpg                         1
+sync_device                 1
+ingredient_substitution     1
+Name: count, dtype: int64
+
+Classification report saved to classification_report_llama3.2:3b_oos30notoos.txt
+
+
+
+
+
+
+================================================================================
+
+
+Run: 40notoos
+Overall Accuracy: 17.00%
+Overall F1: 29.06%
+                       precision    recall  f1-score   support
+
+   what_can_i_ask_you       0.00       nan      0.00         0
+     ingredients_list       0.00       nan      0.00         0
+    travel_suggestion       0.00       nan      0.00         0
+restaurant_suggestion       0.00       nan      0.00         0
+         travel_alert       0.00       nan      0.00         0
+             calories       0.00       nan      0.00         0
+           calculator       0.00       nan      0.00         0
+           definition       0.00       nan      0.00         0
+                  oos       1.00      0.17      0.29       100
+             fun_fact       0.00       nan      0.00         0
+     current_location       0.00       nan      0.00         0
+           book_hotel       0.00       nan      0.00         0
+   international_fees       0.00       nan      0.00         0
+         lost_luggage       0.00       nan      0.00         0
+                  mpg       0.00       nan      0.00         0
+             pay_bill       0.00       nan      0.00         0
+                maybe       0.00       nan      0.00         0
+        change_volume       0.00       nan      0.00         0
+
+             accuracy                           0.17       100
+            macro avg       0.06      0.17      0.02       100
+         weighted avg       1.00      0.17      0.29       100
+
+predicted
+definition               23
+travel_suggestion        17
+oos                      17
+international_fees        7
+fun_fact                  5
+travel_alert              5
+book_hotel                4
+restaurant_suggestion     4
+calories                  4
+current_location          3
+maybe                     2
+ingredients_list          2
+mpg                       2
+what_can_i_ask_you        1
+calculator                1
+lost_luggage              1
+pay_bill                  1
+change_volume             1
+Name: count, dtype: int64
+
+Classification report saved to classification_report_llama3.2:3b_oos40notoos.txt
+
+
+
+
+
+
+================================================================================
+
+
+Run: 50notoos
+Overall Accuracy: 35.00%
+Overall F1: 51.85%
+                       precision    recall  f1-score   support
+
+             fun_fact       0.00       nan      0.00         0
+                  oos       1.00      0.35      0.52       100
+           book_hotel       0.00       nan      0.00         0
+restaurant_suggestion       0.00       nan      0.00         0
+    travel_suggestion       0.00       nan      0.00         0
+  credit_limit_change       0.00       nan      0.00         0
+             calories       0.00       nan      0.00         0
+           definition       0.00       nan      0.00         0
+     current_location       0.00       nan      0.00         0
+                  mpg       0.00       nan      0.00         0
+ schedule_maintenance       0.00       nan      0.00         0
+                  gas       0.00       nan      0.00         0
+         travel_alert       0.00       nan      0.00         0
+   application_status       0.00       nan      0.00         0
+                maybe       0.00       nan      0.00         0
+           play_music       0.00       nan      0.00         0
+   international_fees       0.00       nan      0.00         0
+               recipe       0.00       nan      0.00         0
+             how_busy       0.00       nan      0.00         0
+
+             accuracy                           0.35       100
+            macro avg       0.05      0.35      0.03       100
+         weighted avg       1.00      0.35      0.52       100
+
+predicted
+oos                      35
+definition               21
+fun_fact                  6
+travel_suggestion         6
+book_hotel                5
+calories                  5
+schedule_maintenance      4
+mpg                       2
+restaurant_suggestion     2
+application_status        2
+travel_alert              2
+recipe                    2
+international_fees        2
+current_location          1
+credit_limit_change       1
+maybe                     1
+gas                       1
+play_music                1
+how_busy                  1
+Name: count, dtype: int64
+
+Classification report saved to classification_report_llama3.2:3b_oos50notoos.txt
+
+
+
+
+
+
+================================================================================
+
+
+
+Run: 75notoos
+Overall Accuracy: 14.00%
+Overall F1: 24.56%
+                       precision    recall  f1-score   support
+
+             fun_fact       0.00       nan      0.00         0
+ shopping_list_update       0.00       nan      0.00         0
+    travel_suggestion       0.00       nan      0.00         0
+restaurant_suggestion       0.00       nan      0.00         0
+  credit_limit_change       0.00       nan      0.00         0
+           definition       0.00       nan      0.00         0
+        tire_pressure       0.00       nan      0.00         0
+                  oos       1.00      0.14      0.25       100
+                 date       0.00       nan      0.00         0
+ schedule_maintenance       0.00       nan      0.00         0
+               recipe       0.00       nan      0.00         0
+           find_phone       0.00       nan      0.00         0
+            cook_time       0.00       nan      0.00         0
+                   w2       0.00       nan      0.00         0
+           book_hotel       0.00       nan      0.00         0
+           calculator       0.00       nan      0.00         0
+                  gas       0.00       nan      0.00         0
+         order_status       0.00       nan      0.00         0
+         travel_alert       0.00       nan      0.00         0
+                maybe       0.00       nan      0.00         0
+                  mpg       0.00       nan      0.00         0
+           play_music       0.00       nan      0.00         0
+   application_status       0.00       nan      0.00         0
+   international_fees       0.00       nan      0.00         0
+             how_busy       0.00       nan      0.00         0
+             reminder       0.00       nan      0.00         0
+
+             accuracy                           0.14       100
+            macro avg       0.04      0.14      0.01       100
+         weighted avg       1.00      0.14      0.25       100
+
+predicted
+definition               21
+travel_suggestion        16
+oos                      14
+fun_fact                  8
+restaurant_suggestion     6
+recipe                    5
+book_hotel                3
+credit_limit_change       3
+shopping_list_update      2
+tire_pressure             2
+play_music                2
+schedule_maintenance      2
+calculator                2
+find_phone                2
+date                      1
+w2                        1
+gas                       1
+cook_time                 1
+order_status              1
+travel_alert              1
+maybe                     1
+mpg                       1
+application_status        1
+international_fees        1
+how_busy                  1
+reminder                  1
+Name: count, dtype: int64
+
+Classification report saved to classification_report_llama3.2:3b_oos75notoos.txt
+
+
+================================================================================