Upload 2 files

predictions-zeroshot/round8-fewshot-1exampleeach-k-knownintent-restoos-100oossentences/classification_report_llama3.2_3b_banking_full.txt

@@ -0,0 +1,388 @@
+Run: 1notoos
+Overall Accuracy: 100.00%
+Overall F1: 100.00%
+              precision    recall  f1-score   support
+
+         oos       1.00      1.00      1.00       100
+
+    accuracy                           1.00       100
+   macro avg       1.00      1.00      1.00       100
+weighted avg       1.00      1.00      1.00       100
+
+predicted
+oos    100
+Name: count, dtype: int64
+
+Classification report saved to classification_report_llama3.2:3b_banking1notoos.txt
+
+
+================================================================================
+
+
+Run: 5notoos
+Overall Accuracy: 15.00%
+Overall F1: 26.09%
+                          precision    recall  f1-score   support
+
+                     oos       1.00      0.15      0.26       100
+virtual_card_not_working       0.00       nan      0.00         0
+     why_verify_identity       0.00       nan      0.00         0
+
+                accuracy                           0.15       100
+               macro avg       0.33      0.15      0.09       100
+            weighted avg       1.00      0.15      0.26       100
+
+predicted
+virtual_card_not_working    83
+oos                         15
+why_verify_identity          2
+Name: count, dtype: int64
+
+Classification report saved to classification_report_llama3.2:3b_banking5notoos.txt
+
+
+
+
+
+
+================================================================================
+
+
+
+
+Run: 10notoos
+Overall Accuracy: 43.00%
+Overall F1: 60.14%
+                           precision    recall  f1-score   support
+
+                      oos       1.00      0.43      0.60       100
+unable_to_verify_identity       0.00       nan      0.00         0
+ virtual_card_not_working       0.00       nan      0.00         0
+          transfer_timing       0.00       nan      0.00         0
+       verify_my_identity       0.00       nan      0.00         0
+
+                 accuracy                           0.43       100
+                macro avg       0.20      0.43      0.12       100
+             weighted avg       1.00      0.43      0.60       100
+
+predicted
+oos                          43
+virtual_card_not_working     22
+verify_my_identity           21
+transfer_timing               9
+unable_to_verify_identity     5
+Name: count, dtype: int64
+
+Classification report saved to classification_report_llama3.2:3b_banking10notoos.txt
+
+
+================================================================================
+
+
+Run: 15notoos
+Overall Accuracy: 4.00%
+Overall F1: 7.69%
+                           precision    recall  f1-score   support
+
+unable_to_verify_identity       0.00       nan      0.00         0
+ virtual_card_not_working       0.00       nan      0.00         0
+       topping_up_by_card       0.00       nan      0.00         0
+          transfer_timing       0.00       nan      0.00         0
+                      oos       1.00      0.04      0.08       100
+       verify_my_identity       0.00       nan      0.00         0
+
+                 accuracy                           0.04       100
+                macro avg       0.17      0.04      0.01       100
+             weighted avg       1.00      0.04      0.08       100
+
+predicted
+virtual_card_not_working     62
+topping_up_by_card           23
+unable_to_verify_identity     5
+transfer_timing               4
+oos                           4
+verify_my_identity            2
+Name: count, dtype: int64
+
+Classification report saved to classification_report_llama3.2:3b_banking15notoos.txt
+
+
+================================================================================
+
+
+Run: 20notoos
+Overall Accuracy: 13.00%
+Overall F1: 23.01%
+                           precision    recall  f1-score   support
+
+       topping_up_by_card       0.00       nan      0.00         0
+                      oos       1.00      0.13      0.23       100
+ virtual_card_not_working       0.00       nan      0.00         0
+unable_to_verify_identity       0.00       nan      0.00         0
+            top_up_failed       0.00       nan      0.00         0
+    transfer_into_account       0.00       nan      0.00         0
+    top_up_by_card_charge       0.00       nan      0.00         0
+       verify_my_identity       0.00       nan      0.00         0
+          transfer_timing       0.00       nan      0.00         0
+
+                 accuracy                           0.13       100
+                macro avg       0.11      0.13      0.03       100
+             weighted avg       1.00      0.13      0.23       100
+
+predicted
+topping_up_by_card           49
+virtual_card_not_working     19
+oos                          13
+unable_to_verify_identity    12
+top_up_by_card_charge         2
+verify_my_identity            2
+top_up_failed                 1
+transfer_into_account         1
+transfer_timing               1
+Name: count, dtype: int64
+
+Classification report saved to classification_report_llama3.2:3b_banking20notoos.txt
+
+
+
+================================================================================
+
+
+Run: 30notoos
+Overall Accuracy: 10.00%
+Overall F1: 18.18%
+                                precision    recall  f1-score   support
+
+supported_cards_and_currencies       0.00       nan      0.00         0
+            topping_up_by_card       0.00       nan      0.00         0
+                   pin_blocked       0.00       nan      0.00         0
+     unable_to_verify_identity       0.00       nan      0.00         0
+      virtual_card_not_working       0.00       nan      0.00         0
+            verify_my_identity       0.00       nan      0.00         0
+                           oos       1.00      0.10      0.18       100
+                 top_up_failed       0.00       nan      0.00         0
+         top_up_by_card_charge       0.00       nan      0.00         0
+             terminate_account       0.00       nan      0.00         0
+
+                      accuracy                           0.10       100
+                     macro avg       0.10      0.10      0.02       100
+                  weighted avg       1.00      0.10      0.18       100
+
+predicted
+topping_up_by_card                47
+pin_blocked                       16
+oos                               10
+virtual_card_not_working           9
+supported_cards_and_currencies     5
+unable_to_verify_identity          4
+verify_my_identity                 4
+top_up_failed                      2
+top_up_by_card_charge              2
+terminate_account                  1
+Name: count, dtype: int64
+
+Classification report saved to classification_report_llama3.2:3b_banking30notoos.txt
+
+
+
+================================================================================
+
+
+Run: 35notoos
+Overall Accuracy: 2.00%
+Overall F1: 3.92%
+                          precision    recall  f1-score   support
+
+     order_physical_card       0.00       nan      0.00         0
+      passcode_forgotten       0.00       nan      0.00         0
+      topping_up_by_card       0.00       nan      0.00         0
+             pin_blocked       0.00       nan      0.00         0
+      verify_my_identity       0.00       nan      0.00         0
+virtual_card_not_working       0.00       nan      0.00         0
+                     oos       1.00      0.02      0.04       100
+    pending_card_payment       0.00       nan      0.00         0
+    lost_or_stolen_phone       0.00       nan      0.00         0
+
+                accuracy                           0.02       100
+               macro avg       0.11      0.02      0.00       100
+            weighted avg       1.00      0.02      0.04       100
+
+predicted
+order_physical_card         71
+passcode_forgotten           8
+virtual_card_not_working     8
+pin_blocked                  5
+topping_up_by_card           3
+oos                          2
+verify_my_identity           1
+pending_card_payment         1
+lost_or_stolen_phone         1
+Name: count, dtype: int64
+
+Classification report saved to classification_report_llama3.2:3b_banking35notoos.txt
+
+
+
+
+
+================================================================================
+
+
+Run: 40notoos
+Overall Accuracy: 2.00%
+Overall F1: 3.92%
+                           precision    recall  f1-score   support
+
+     getting_virtual_card       0.00       nan      0.00         0
+        get_physical_card       0.00       nan      0.00         0
+      order_physical_card       0.00       nan      0.00         0
+       verify_my_identity       0.00       nan      0.00         0
+unable_to_verify_identity       0.00       nan      0.00         0
+                      oos       1.00      0.02      0.04       100
+ virtual_card_not_working       0.00       nan      0.00         0
+      lost_or_stolen_card       0.00       nan      0.00         0
+
+                 accuracy                           0.02       100
+                macro avg       0.12      0.02      0.00       100
+             weighted avg       1.00      0.02      0.04       100
+
+predicted
+get_physical_card            67
+getting_virtual_card         17
+order_physical_card           9
+oos                           2
+virtual_card_not_working      2
+verify_my_identity            1
+unable_to_verify_identity     1
+lost_or_stolen_card           1
+Name: count, dtype: int64
+
+Classification report saved to classification_report_llama3.2:3b_banking40notoos.txt
+
+
+
+
+================================================================================
+
+Run: 50notoos
+Overall Accuracy: 4.00%
+Overall F1: 7.69%
+                                     precision    recall  f1-score   support
+
+                order_physical_card       0.00       nan      0.00         0
+                  get_physical_card       0.00       nan      0.00         0
+               getting_virtual_card       0.00       nan      0.00         0
+direct_debit_payment_not_recognised       0.00       nan      0.00         0
+              edit_personal_details       0.00       nan      0.00         0
+                 verify_my_identity       0.00       nan      0.00         0
+                        pin_blocked       0.00       nan      0.00         0
+                                oos       1.00      0.04      0.08       100
+           virtual_card_not_working       0.00       nan      0.00         0
+                lost_or_stolen_card       0.00       nan      0.00         0
+
+                           accuracy                           0.04       100
+                          macro avg       0.10      0.04      0.01       100
+                       weighted avg       1.00      0.04      0.08       100
+
+predicted
+order_physical_card                    43
+get_physical_card                      25
+getting_virtual_card                   15
+edit_personal_details                   6
+oos                                     4
+pin_blocked                             2
+virtual_card_not_working                2
+direct_debit_payment_not_recognised     1
+verify_my_identity                      1
+lost_or_stolen_card                     1
+Name: count, dtype: int64
+
+Classification report saved to classification_report_llama3.2:3b_banking50notoos.txt
+
+
+================================================================================
+
+Run: 60notoos
+Overall Accuracy: 2.00%
+Overall F1: 3.92%
+                           precision    recall  f1-score   support
+
+      order_physical_card       0.00       nan      0.00         0
+       getting_spare_card       0.00       nan      0.00         0
+     getting_virtual_card       0.00       nan      0.00         0
+        get_physical_card       0.00       nan      0.00         0
+           card_swallowed       0.00       nan      0.00         0
+               change_pin       0.00       nan      0.00         0
+  contactless_not_working       0.00       nan      0.00         0
+       verify_my_identity       0.00       nan      0.00         0
+unable_to_verify_identity       0.00       nan      0.00         0
+                      oos       1.00      0.02      0.04       100
+              pin_blocked       0.00       nan      0.00         0
+ virtual_card_not_working       0.00       nan      0.00         0
+      lost_or_stolen_card       0.00       nan      0.00         0
+
+                 accuracy                           0.02       100
+                macro avg       0.08      0.02      0.00       100
+             weighted avg       1.00      0.02      0.04       100
+
+predicted
+order_physical_card          36
+getting_virtual_card         35
+getting_spare_card            7
+get_physical_card             7
+contactless_not_working       5
+card_swallowed                2
+oos                           2
+verify_my_identity            1
+change_pin                    1
+unable_to_verify_identity     1
+pin_blocked                   1
+virtual_card_not_working      1
+lost_or_stolen_card           1
+Name: count, dtype: int64
+
+Classification report saved to classification_report_llama3.2:3b_banking60notoos.txt
+
+
+================================================================================
+
+Run: 70notoos
+Overall Accuracy: 2.00%
+Overall F1: 3.92%
+                           precision    recall  f1-score   support
+
+          card_acceptance       0.00       nan      0.00         0
+      order_physical_card       0.00       nan      0.00         0
+             card_linking       0.00       nan      0.00         0
+             card_arrival       0.00       nan      0.00         0
+         card_not_working       0.00       nan      0.00         0
+    edit_personal_details       0.00       nan      0.00         0
+  contactless_not_working       0.00       nan      0.00         0
+unable_to_verify_identity       0.00       nan      0.00         0
+    Refund_not_showing_up       0.00       nan      0.00         0
+                      oos       1.00      0.02      0.04       100
+      lost_or_stolen_card       0.00       nan      0.00         0
+
+                 accuracy                           0.02       100
+                macro avg       0.09      0.02      0.00       100
+             weighted avg       1.00      0.02      0.04       100
+
+predicted
+order_physical_card          36
+card_acceptance              23
+card_arrival                 20
+card_not_working              9
+edit_personal_details         4
+card_linking                  2
+oos                           2
+contactless_not_working       1
+unable_to_verify_identity     1
+Refund_not_showing_up         1
+lost_or_stolen_card           1
+Name: count, dtype: int64
+
+Classification report saved to classification_report_llama3.2:3b_banking70notoos.txt
+
+
+
+================================================================================

predictions-zeroshot/round8-fewshot-1exampleeach-k-knownintent-restoos-100oossentences/classification_report_llama3.2_3b_stackoverflow_full.txt

@@ -0,0 +1,299 @@
+Run: 1notoos
+Overall Accuracy: 100.00%
+Overall F1: 100.00%
+              precision    recall  f1-score   support
+
+         oos       1.00      1.00      1.00       100
+
+    accuracy                           1.00       100
+   macro avg       1.00      1.00      1.00       100
+weighted avg       1.00      1.00      1.00       100
+
+predicted
+oos    100
+Name: count, dtype: int64
+
+Classification report saved to classification_report_llama3.2:3b_stackoverflow1notoos.txt
+
+
+
+================================================================================
+
+
+
+Run: 2notoos
+Overall Accuracy: 94.00%
+Overall F1: 96.91%
+              precision    recall  f1-score   support
+
+         oos       1.00      0.94      0.97       100
+     magento       0.00       nan      0.00         0
+
+    accuracy                           0.94       100
+   macro avg       0.50      0.94      0.48       100
+weighted avg       1.00      0.94      0.97       100
+
+predicted
+oos        94
+magento     6
+Name: count, dtype: int64
+
+Classification report saved to classification_report_llama3.2:3b_stackoverflow2notoos.txt
+
+================================================================================
+
+Run: 4notoos
+Overall Accuracy: 81.00%
+Overall F1: 89.50%
+              precision    recall  f1-score   support
+
+         oos       1.00      0.81      0.90       100
+        linq       0.00       nan      0.00         0
+      drupal       0.00       nan      0.00         0
+     magento       0.00       nan      0.00         0
+
+    accuracy                           0.81       100
+   macro avg       0.25      0.81      0.22       100
+weighted avg       1.00      0.81      0.90       100
+
+predicted
+oos        81
+linq        8
+magento     8
+drupal      3
+Name: count, dtype: int64
+
+Classification report saved to classification_report_llama3.2:3b_stackoverflow4notoos.txt
+
+
+================================================================================
+
+Run: 6notoos
+Overall Accuracy: 25.00%
+Overall F1: 40.00%
+              precision    recall  f1-score   support
+
+     magento       0.00       nan      0.00         0
+         oos       1.00      0.25      0.40       100
+        linq       0.00       nan      0.00         0
+      drupal       0.00       nan      0.00         0
+        ajax       0.00       nan      0.00         0
+
+    accuracy                           0.25       100
+   macro avg       0.20      0.25      0.08       100
+weighted avg       1.00      0.25      0.40       100
+
+predicted
+magento    53
+oos        25
+drupal     14
+linq        7
+ajax        1
+Name: count, dtype: int64
+
+Classification report saved to classification_report_llama3.2:3b_stackoverflow6notoos.txt
+
+
+================================================================================
+
+Run: 8notoos
+Overall Accuracy: 17.00%
+Overall F1: 29.06%
+              precision    recall  f1-score   support
+
+     magento       0.00       nan      0.00         0
+      drupal       0.00       nan      0.00         0
+        linq       0.00       nan      0.00         0
+        ajax       0.00       nan      0.00         0
+         oos       1.00      0.17      0.29       100
+  sharepoint       0.00       nan      0.00         0
+
+    accuracy                           0.17       100
+   macro avg       0.17      0.17      0.05       100
+weighted avg       1.00      0.17      0.29       100
+
+predicted
+magento       46
+oos           17
+sharepoint    12
+linq          11
+drupal        10
+ajax           4
+Name: count, dtype: int64
+
+Classification report saved to classification_report_llama3.2:3b_stackoverflow8notoos.txt
+
+
+================================================================================
+
+Run: 10notoos
+Overall Accuracy: 61.00%
+Overall F1: 75.78%
+              precision    recall  f1-score   support
+
+         oos       1.00      0.61      0.76       100
+  sharepoint       0.00       nan      0.00         0
+        linq       0.00       nan      0.00         0
+      drupal       0.00       nan      0.00         0
+     magento       0.00       nan      0.00         0
+        ajax       0.00       nan      0.00         0
+
+    accuracy                           0.61       100
+   macro avg       0.17      0.61      0.13       100
+weighted avg       1.00      0.61      0.76       100
+
+predicted
+oos           61
+sharepoint    21
+linq           7
+magento        6
+ajax           3
+drupal         2
+Name: count, dtype: int64
+
+Classification report saved to classification_report_llama3.2:3b_stackoverflow10notoos.txt
+
+
+================================================================================
+
+
+Run: 12notoos
+Overall Accuracy: 11.00%
+Overall F1: 19.82%
+              precision    recall  f1-score   support
+
+     magento       0.00       nan      0.00         0
+  sharepoint       0.00       nan      0.00         0
+         oos       1.00      0.11      0.20       100
+      drupal       0.00       nan      0.00         0
+        bash       0.00       nan      0.00         0
+        linq       0.00       nan      0.00         0
+        ajax       0.00       nan      0.00         0
+
+    accuracy                           0.11       100
+   macro avg       0.14      0.11      0.03       100
+weighted avg       1.00      0.11      0.20       100
+
+predicted
+sharepoint    39
+magento       33
+oos           11
+drupal         8
+bash           4
+linq           3
+ajax           2
+Name: count, dtype: int64
+
+Classification report saved to classification_report_llama3.2:3b_stackoverflow12notoos.txt
+
+
+================================================================================
+
+Run: 14notoos
+Overall Accuracy: 8.00%
+Overall F1: 14.81%
+               precision    recall  f1-score   support
+
+visual-studio       0.00       nan      0.00         0
+      magento       0.00       nan      0.00         0
+         linq       0.00       nan      0.00         0
+          oos       1.00      0.08      0.15       100
+       drupal       0.00       nan      0.00         0
+   sharepoint       0.00       nan      0.00         0
+        cocoa       0.00       nan      0.00         0
+         ajax       0.00       nan      0.00         0
+         bash       0.00       nan      0.00         0
+
+     accuracy                           0.08       100
+    macro avg       0.11      0.08      0.02       100
+ weighted avg       1.00      0.08      0.15       100
+
+predicted
+visual-studio    48
+sharepoint       15
+magento          13
+oos               8
+drupal            6
+linq              4
+cocoa             4
+ajax              1
+bash              1
+Name: count, dtype: int64
+
+Classification report saved to classification_report_llama3.2:3b_stackoverflow14notoos.txt
+
+
+================================================================================
+
+Run: 16notoos
+Overall Accuracy: 7.00%
+Overall F1: 13.08%
+               precision    recall  f1-score   support
+
+visual-studio       0.00       nan      0.00         0
+       drupal       0.00       nan      0.00         0
+        cocoa       0.00       nan      0.00         0
+         ajax       0.00       nan      0.00         0
+          oos       1.00      0.07      0.13       100
+   sharepoint       0.00       nan      0.00         0
+      magento       0.00       nan      0.00         0
+        excel       0.00       nan      0.00         0
+         bash       0.00       nan      0.00         0
+
+     accuracy                           0.07       100
+    macro avg       0.11      0.07      0.01       100
+ weighted avg       1.00      0.07      0.13       100
+
+predicted
+drupal           52
+cocoa            19
+oos               7
+visual-studio     6
+sharepoint        6
+ajax              5
+magento           2
+excel             2
+bash              1
+Name: count, dtype: int64
+
+Classification report saved to classification_report_llama3.2:3b_stackoverflow16notoos.txt
+
+
+================================================================================
+
+Run: 18notoos
+Overall Accuracy: 3.00%
+Overall F1: 5.83%
+               precision    recall  f1-score   support
+
+       apache       0.00       nan      0.00         0
+       drupal       0.00       nan      0.00         0
+        excel       0.00       nan      0.00         0
+         ajax       0.00       nan      0.00         0
+visual-studio       0.00       nan      0.00         0
+          oos       1.00      0.03      0.06       100
+   sharepoint       0.00       nan      0.00         0
+        cocoa       0.00       nan      0.00         0
+      magento       0.00       nan      0.00         0
+
+     accuracy                           0.03       100
+    macro avg       0.11      0.03      0.01       100
+ weighted avg       1.00      0.03      0.06       100
+
+predicted
+drupal           41
+apache           20
+excel            17
+sharepoint        6
+ajax              4
+visual-studio     4
+oos               3
+cocoa             3
+magento           2
+Name: count, dtype: int64
+
+Classification report saved to classification_report_llama3.2:3b_stackoverflow18notoos.txt
+
+
+
+================================================================================