complete finetune rpp results

llm_toolkit/translation_utils.py

@@ -108,7 +108,7 @@ def get_few_shot_prompt(dataset, num_shots=5):
     return translation_prompt
 
 
-def load_translation_dataset(data_path, tokenizer=None, num_shots=5):
+def load_translation_dataset(data_path, tokenizer=None, num_shots=0):
     train_data_file = data_path.replace(".tsv", "-train.tsv")
     test_data_file = data_path.replace(".tsv", "-test.tsv")
 

logs/l40-1gpu-rpp.txt

@@ -235,4 +235,17 @@ internlm/internlm2_5-7b-chat/checkpoint-140/rpp-1.08 metrics: {'meteor': 0.38098
 Batch output: ['Old Geng raised his rifle, squinted one eye shut, and fired off a shot which sounded like hailstones falling from the sky.', 'The next morning, before dawn, Grannie Liu got up early for her bath and toilet routine. She then drilled Ban-er on some phrases so that when he accompanied her into town, he could answer questions put to him. A five-year-old boy would have been delighted at the prospect of going into the city!']
 internlm/internlm2_5-7b-chat/checkpoint-140/rpp-1.10 metrics: {'meteor': 0.3650187980362992, 'bleu_scores': {'bleu': 0.09961847338588378, 'precisions': [0.4301199865435652, 0.1539422326307572, 0.06998775010208248, 0.035527780155808576], 'brevity_penalty': 0.8794385616155197, 'length_ratio': 0.886154355746936, 'translation_length': 26753, 'reference_length': 30190}, 'rouge_scores': {'rouge1': 0.4038157103042593, 'rouge2': 0.16595825722923374, 'rougeL': 0.34796330037067624, 'rougeLsum': 0.34814873412639924}, 'accuracy': 0.00706090026478376, 'correct_ids': [170, 240, 309, 348, 533, 658, 659, 1011]}
 *** Evaluating with repetition_penalty: 1.12
-
  0%|          | 0/567 [00:00<?, ?it/s]
  0%|          | 1/567 [00:11<1:44:52, 11.12s/it]
  0%|          | 2/567 [00:23<1:50:40, 11.75s/it]
  1%|          | 3/567 [00:29<1:26:57,  9.25s/it]
  1%|          | 4/567 [00:45<1:49:51, 11.71s/it]
  1%|          | 5/567 [00:47<1:18:48,  8.41s/it]
  1%|          | 6/567 [00:52<1:08:50,  7.36s/it]
  1%|          | 7/567 [00:55<53:37,  5.75s/it]  
  1%|▏         | 8/567 [01:02<57:21,  6.16s/it]
  2%|▏         | 9/567 [01:13<1:11:03,  7.64s/it]
  2%|▏         | 10/567 [01:19<1:05:31,  7.06s/it]
  2%|▏         | 11/567 [01:25<1:03:54,  6.90s/it]
  2%|▏         | 12/567 [01:28<52:54,  5.72s/it]  
  2%|▏         | 13/567 [01:35<55:42,  6.03s/it]
  2%|▏         | 14/567 [01:40<52:13,  5.67s/it]
  3%|▎         | 15/567 [01:42<41:32,  4.52s/it]
  3%|▎         | 16/567 [01:49<50:10,  5.46s/it]
  3%|▎         | 17/567 [01:55<52:09,  5.69s/it]
  3%|▎         | 18/567 [01:59<46:56,  5.13s/it]
  3%|▎         | 19/567 [02:02<40:39,  4.45s/it]
  4%|▎         | 20/567 [02:06<38:20,  4.21s/it]
  4%|▎         | 21/567 [02:12<43:09,  4.74s/it]
  4%|▍         | 22/567 [02:16<41:53,  4.61s/it]
  4%|▍         | 23/567 [02:23<47:40,  5.26s/it]
  4%|▍         | 24/567 [02:26<43:16,  4.78s/it]
  4%|▍         | 25/567 [02:29<36:49,  4.08s/it]
  5%|▍         | 26/567 [02:32<35:17,  3.91s/it]
  5%|▍         | 27/567 [02:35<31:37,  3.51s/it]
  5%|▍         | 28/567 [02:47<55:12,  6.15s/it]
  5%|▌         | 29/567 [02:53<52:45,  5.88s/it]
  5%|▌         | 30/567 [03:00<57:01,  6.37s/it]
  5%|▌         | 31/567 [03:03<46:42,  5.23s/it]
  6%|▌         | 32/567 [03:08<47:01,  5.27s/it]
  6%|▌         | 33/567 [03:13<45:07,  5.07s/it]
  6%|▌         | 34/567 [03:19<49:11,  5.54s/it]
  6%|▌         | 35/567 [03:26<51:08,  5.77s/it]
  6%|▋         | 36/567 [03:28<41:24,  4.68s/it]
  7%|▋         | 37/567 [03:30<33:50,  3.83s/it]
  7%|▋         | 38/567 [03:33<32:00,  3.63s/it]
  7%|▋         | 39/567 [03:37<34:09,  3.88s/it]
  7%|▋         | 40/567 [03:44<40:32,  4.62s/it]
  7%|▋         | 41/567 [03:45<32:05,  3.66s/it]
  7%|▋         | 42/567 [03:57<55:14,  6.31s/it]
  8%|▊         | 43/567 [04:00<46:05,  5.28s/it]
  8%|▊         | 44/567 [04:06<46:48,  5.37s/it]
  8%|▊         | 45/567 [04:13<52:11,  6.00s/it]
  8%|▊         | 46/567 [04:16<42:48,  4.93s/it]
  8%|▊         | 47/567 [04:19<39:21,  4.54s/it]
  8%|▊         | 48/567 [04:25<42:22,  4.90s/it]
  9%|▊         | 49/567 [04:28<37:00,  4.29s/it]
  9%|▉         | 50/567 [04:36<46:58,  5.45s/it]
  9%|▉         | 51/567 [04:44<52:33,  6.11s/it]
  9%|▉         | 52/567 [04:45<40:45,  4.75s/it]
  9%|▉         | 53/567 [05:08<1:25:12,  9.95s/it]
 10%|▉         | 54/567 [05:12<1:11:23,  8.35s/it]
 10%|▉         | 55/567 [05:21<1:12:29,  8.50s/it]
 10%|▉         | 56/567 [05:34<1:25:04,  9.99s/it]
 10%|█         | 57/567 [05:46<1:28:37, 10.43s/it]
 10%|█         | 58/567 [05:49<1:09:10,  8.15s/it]
 10%|█         | 59/567 [05:55<1:03:32,  7.51s/it]
 11%|█         | 60/567 [06:00<58:18,  6.90s/it]  
 11%|█         | 61/567 [06:04<49:28,  5.87s/it]
 11%|█         | 62/567 [06:08<45:20,  5.39s/it]
 11%|█         | 63/567 [06:14<47:35,  5.67s/it]
 11%|█▏        | 64/567 [06:23<54:33,  6.51s/it]
 11%|█▏        | 65/567 [06:25<42:46,  5.11s/it]
 12%|█▏        | 66/567 [06:34<54:06,  6.48s/it]
 12%|█▏        | 67/567 [06:37<44:32,  5.34s/it]
 12%|█▏        | 68/567 [06:51<1:06:51,  8.04s/it]
 12%|█▏        | 69/567 [06:55<55:32,  6.69s/it]  
 12%|█▏        | 70/567 [07:01<53:23,  6.45s/it]
 13%|█▎        | 71/567 [07:04<45:07,  5.46s/it]
 13%|█▎        | 72/567 [07:08<42:46,  5.18s/it]
 13%|█▎        | 73/567 [07:12<38:00,  4.62s/it]
 13%|█▎        | 74/567 [07:17<39:01,  4.75s/it]
 13%|█▎        | 75/567 [07:20<35:24,  4.32s/it]
 13%|█▎        | 76/567 [07:25<37:29,  4.58s/it]
 14%|█▎        | 77/567 [07:34<47:42,  5.84s/it]
 14%|█▍        | 78/567 [07:39<45:00,  5.52s/it]
 14%|█▍        | 79/567 [07:44<43:23,  5.34s/it]
 14%|█▍        | 80/567 [07:47<38:42,  4.77s/it]
 14%|█▍        | 81/567 [07:51<35:46,  4.42s/it]
 14%|█▍        | 82/567 [07:55<34:22,  4.25s/it]
 15%|█▍        | 83/567 [07:59<33:44,  4.18s/it]
 15%|█▍        | 84/567 [08:02<31:33,  3.92s/it]
 15%|█▍        | 85/567 [08:04<25:50,  3.22s/it]
 15%|█▌        | 86/567 [08:05<22:08,  2.76s/it]
 15%|█▌        | 87/567 [08:08<21:37,  2.70s/it]
 16%|█▌        | 88/567 [08:12<24:32,  3.07s/it]
 16%|█▌        | 89/567 [08:25<47:53,  6.01s/it]
 16%|█▌        | 90/567 [08:30<46:54,  5.90s/it]
 16%|█▌        | 91/567 [08:37<48:38,  6.13s/it]
 16%|█▌        | 92/567 [08:39<39:05,  4.94s/it]
 16%|█▋        | 93/567 [08:41<32:42,  4.14s/it]
 17%|█▋        | 94/567 [08:50<42:38,  5.41s/it]
 17%|█▋        | 95/567 [08:53<36:32,  4.64s/it]
 17%|█▋        | 96/567 [08:58<37:33,  4.78s/it]
 17%|█▋        | 97/567 [09:01<33:35,  4.29s/it]
 17%|█▋        | 98/567 [09:03<28:49,  3.69s/it]
 17%|█▋        | 99/567 [09:07<28:44,  3.68s/it]
 18%|█▊        | 100/567 [09:15<39:54,  5.13s/it]
 18%|█▊        | 101/567 [09:19<36:47,  4.74s/it]
 18%|█▊        | 102/567 [09:28<47:02,  6.07s/it]
 18%|█▊        | 103/567 [09:30<36:09,  4.68s/it]
 18%|█▊        | 104/567 [09:38<43:25,  5.63s/it]
 19%|█▊        | 105/567 [09:51<1:01:29,  7.99s/it]
 19%|█▊        | 106/567 [09:58<1:00:02,  7.81s/it]
 19%|█▉        | 107/567 [10:01<48:50,  6.37s/it]  
 19%|█▉        | 108/567 [10:17<1:10:13,  9.18s/it]
 19%|█▉        | 109/567 [10:19<52:19,  6.86s/it]  
 19%|█▉        | 110/567 [10:22<44:27,  5.84s/it]
 20%|█▉        | 111/567 [10:36<1:03:31,  8.36s/it]
 20%|█▉        | 112/567 [10:44<1:01:33,  8.12s/it]
 20%|█▉        | 113/567 [10:56<1:10:17,  9.29s/it]
 20%|██        | 114/567 [11:00<58:37,  7.76s/it]  
 20%|██        | 115/567 [11:05<51:37,  6.85s/it]
 20%|██        | 116/567 [11:08<42:56,  5.71s/it]
 21%|██        | 117/567 [11:12<38:39,  5.15s/it]
 21%|██        | 118/567 [11:14<32:54,  4.40s/it]
 21%|██        | 119/567 [11:25<47:30,  6.36s/it]
 21%|██        | 120/567 [11:29<41:38,  5.59s/it]
 21%|██▏       | 121/567 [11:32<34:46,  4.68s/it]
 22%|██▏       | 122/567 [11:35<31:03,  4.19s/it]
 22%|██▏       | 123/567 [11:39<31:11,  4.21s/it]
 22%|██▏       | 124/567 [11:42<27:49,  3.77s/it]
 22%|██▏       | 125/567 [11:45<26:43,  3.63s/it]
 22%|██▏       | 126/567 [11:50<29:24,  4.00s/it]
 22%|██▏       | 127/567 [12:00<42:27,  5.79s/it]
 23%|██▎       | 128/567 [12:07<45:49,  6.26s/it]
 23%|██▎       | 129/567 [12:16<50:10,  6.87s/it]
 23%|██▎       | 130/567 [12:27<1:00:37,  8.32s/it]
 23%|██▎       | 131/567 [12:41<1:12:29,  9.98s/it]
 23%|██▎       | 132/567 [12:53<1:15:47, 10.45s/it]
 23%|██▎       | 133/567 [13:10<1:29:51, 12.42s/it]
 24%|██▎       | 134/567 [13:30<1:46:29, 14.76s/it]
 24%|██▍       | 135/567 [13:35<1:25:48, 11.92s/it]
 24%|██▍       | 136/567 [13:41<1:12:25, 10.08s/it]
 24%|██▍       | 137/567 [13:44<57:07,  7.97s/it]  
 24%|██▍       | 138/567 [13:47<46:02,  6.44s/it]
 25%|██▍       | 139/567 [13:51<40:58,  5.75s/it]
 25%|██▍       | 140/567 [13:56<38:39,  5.43s/it]
 25%|██▍       | 141/567 [14:01<38:48,  5.47s/it]
 25%|██▌       | 142/567 [14:04<33:12,  4.69s/it]
 25%|██▌       | 143/567 [14:07<29:18,  4.15s/it]
 25%|██▌       | 144/567 [14:12<31:41,  4.50s/it]
 26%|██▌       | 145/567 [14:18<34:24,  4.89s/it]
 26%|██▌       | 146/567 [14:23<35:09,  5.01s/it]
 26%|██▌       | 147/567 [14:39<58:11,  8.31s/it]
 26%|██▌       | 148/567 [14:45<51:40,  7.40s/it]
 26%|██▋       | 149/567 [14:47<41:45,  5.99s/it]
 26%|██▋       | 150/567 [14:49<31:50,  4.58s/it]
 27%|██▋       | 151/567 [14:50<25:29,  3.68s/it]
 27%|██▋       | 152/567 [14:55<28:11,  4.08s/it]
 27%|██▋       | 153/567 [15:01<32:22,  4.69s/it]
 27%|██▋       | 154/567 [15:04<28:48,  4.19s/it]
 27%|██▋       | 155/567 [15:10<32:11,  4.69s/it]
 28%|██▊       | 156/567 [15:14<30:31,  4.46s/it]
 28%|██▊       | 157/567 [15:18<29:26,  4.31s/it]
 28%|██▊       | 158/567 [15:30<44:04,  6.47s/it]
 28%|██▊       | 159/567 [15:34<38:48,  5.71s/it]
 28%|██▊       | 160/567 [15:44<47:50,  7.05s/it]
 28%|██▊       | 161/567 [15:46<37:10,  5.49s/it]
 29%|██▊       | 162/567 [15:52<39:13,  5.81s/it]
 29%|██▊       | 163/567 [15:55<34:03,  5.06s/it]
 29%|██▉       | 164/567 [17:01<2:36:28, 23.30s/it]
 29%|██▉       | 165/567 [17:07<2:01:37, 18.15s/it]
 29%|██▉       | 166/567 [17:11<1:32:58, 13.91s/it]
 29%|██▉       | 167/567 [17:22<1:26:07, 12.92s/it]
 30%|██▉       | 168/567 [17:26<1:07:29, 10.15s/it]
 30%|██▉       | 169/567 [17:32<58:59,  8.89s/it]  
 30%|██▉       | 170/567 [17:39<54:48,  8.28s/it]
 30%|███       | 171/567 [17:45<50:33,  7.66s/it]
 30%|███       | 172/567 [17:51<47:54,  7.28s/it]
 31%|███       | 173/567 [17:55<40:29,  6.17s/it]
 31%|███       | 174/567 [18:00<38:22,  5.86s/it]
 31%|███       | 175/567 [18:06<38:41,  5.92s/it]
 31%|███       | 176/567 [18:10<34:29,  5.29s/it]
 31%|███       | 177/567 [18:20<43:13,  6.65s/it]
 31%|███▏      | 178/567 [18:23<36:19,  5.60s/it]
 32%|███▏      | 179/567 [18:28<36:10,  5.59s/it]
 32%|███▏      | 180/567 [18:34<35:20,  5.48s/it]
 32%|███▏      | 181/567 [18:39<35:17,  5.49s/it]
 32%|███▏      | 182/567 [18:41<27:55,  4.35s/it]
 32%|███▏      | 183/567 [18:43<23:05,  3.61s/it]
 32%|███▏      | 184/567 [18:46<22:46,  3.57s/it]
 33%|███▎      | 185/567 [18:50<23:17,  3.66s/it]
 33%|███▎      | 186/567 [19:12<58:10,  9.16s/it]
 33%|███▎      | 187/567 [19:16<49:06,  7.75s/it]
 33%|███▎      | 188/567 [19:19<39:25,  6.24s/it]
 33%|███▎      | 189/567 [19:24<37:11,  5
+
  0%|          | 0/567 [00:00<?, ?it/s]
  0%|          | 1/567 [00:11<1:44:52, 11.12s/it]
  0%|          | 2/567 [00:23<1:50:40, 11.75s/it]
  1%|          | 3/567 [00:29<1:26:57,  9.25s/it]
  1%|          | 4/567 [00:45<1:49:51, 11.71s/it]
  1%|          | 5/567 [00:47<1:18:48,  8.41s/it]
  1%|          | 6/567 [00:52<1:08:50,  7.36s/it]
  1%|          | 7/567 [00:55<53:37,  5.75s/it]  
  1%|▏         | 8/567 [01:02<57:21,  6.16s/it]
  2%|▏         | 9/567 [01:13<1:11:03,  7.64s/it]
  2%|▏         | 10/567 [01:19<1:05:31,  7.06s/it]
  2%|▏         | 11/567 [01:25<1:03:54,  6.90s/it]
  2%|▏         | 12/567 [01:28<52:54,  5.72s/it]  
  2%|▏         | 13/567 [01:35<55:42,  6.03s/it]
  2%|▏         | 14/567 [01:40<52:13,  5.67s/it]
  3%|▎         | 15/567 [01:42<41:32,  4.52s/it]
  3%|▎         | 16/567 [01:49<50:10,  5.46s/it]
  3%|▎         | 17/567 [01:55<52:09,  5.69s/it]
  3%|▎         | 18/567 [01:59<46:56,  5.13s/it]
  3%|▎         | 19/567 [02:02<40:39,  4.45s/it]
  4%|▎         | 20/567 [02:06<38:20,  4.21s/it]
  4%|▎         | 21/567 [02:12<43:09,  4.74s/it]
  4%|▍         | 22/567 [02:16<41:53,  4.61s/it]
  4%|▍         | 23/567 [02:23<47:40,  5.26s/it]
  4%|▍         | 24/567 [02:26<43:16,  4.78s/it]
  4%|▍         | 25/567 [02:29<36:49,  4.08s/it]
  5%|▍         | 26/567 [02:32<35:17,  3.91s/it]
  5%|▍         | 27/567 [02:35<31:37,  3.51s/it]
  5%|▍         | 28/567 [02:47<55:12,  6.15s/it]
  5%|▌         | 29/567 [02:53<52:45,  5.88s/it]
  5%|▌         | 30/567 [03:00<57:01,  6.37s/it]
  5%|▌         | 31/567 [03:03<46:42,  5.23s/it]
  6%|▌         | 32/567 [03:08<47:01,  5.27s/it]
  6%|▌         | 33/567 [03:13<45:07,  5.07s/it]
  6%|▌         | 34/567 [03:19<49:11,  5.54s/it]
  6%|▌         | 35/567 [03:26<51:08,  5.77s/it]
  6%|▋         | 36/567 [03:28<41:24,  4.68s/it]
  7%|▋         | 37/567 [03:30<33:50,  3.83s/it]
  7%|▋         | 38/567 [03:33<32:00,  3.63s/it]
  7%|▋         | 39/567 [03:37<34:09,  3.88s/it]
  7%|▋         | 40/567 [03:44<40:32,  4.62s/it]
  7%|▋         | 41/567 [03:45<32:05,  3.66s/it]
  7%|▋         | 42/567 [03:57<55:14,  6.31s/it]
  8%|▊         | 43/567 [04:00<46:05,  5.28s/it]
  8%|▊         | 44/567 [04:06<46:48,  5.37s/it]
  8%|▊         | 45/567 [04:13<52:11,  6.00s/it]
  8%|▊         | 46/567 [04:16<42:48,  4.93s/it]
  8%|▊         | 47/567 [04:19<39:21,  4.54s/it]
  8%|▊         | 48/567 [04:25<42:22,  4.90s/it]
  9%|▊         | 49/567 [04:28<37:00,  4.29s/it]
  9%|▉         | 50/567 [04:36<46:58,  5.45s/it]
  9%|▉         | 51/567 [04:44<52:33,  6.11s/it]
  9%|▉         | 52/567 [04:45<40:45,  4.75s/it]
  9%|▉         | 53/567 [05:08<1:25:12,  9.95s/it]
 10%|▉         | 54/567 [05:12<1:11:23,  8.35s/it]
 10%|▉         | 55/567 [05:21<1:12:29,  8.50s/it]
 10%|▉         | 56/567 [05:34<1:25:04,  9.99s/it]
 10%|█         | 57/567 [05:46<1:28:37, 10.43s/it]
 10%|█         | 58/567 [05:49<1:09:10,  8.15s/it]
 10%|█         | 59/567 [05:55<1:03:32,  7.51s/it]
 11%|█         | 60/567 [06:00<58:18,  6.90s/it]  
 11%|█         | 61/567 [06:04<49:28,  5.87s/it]
 11%|█         | 62/567 [06:08<45:20,  5.39s/it]
 11%|█         | 63/567 [06:14<47:35,  5.67s/it]
 11%|█▏        | 64/567 [06:23<54:33,  6.51s/it]
 11%|█▏        | 65/567 [06:25<42:46,  5.11s/it]
 12%|█▏        | 66/567 [06:34<54:06,  6.48s/it]
 12%|█▏        | 67/567 [06:37<44:32,  5.34s/it]
 12%|█▏        | 68/567 [06:51<1:06:51,  8.04s/it]
 12%|█▏        | 69/567 [06:55<55:32,  6.69s/it]  
 12%|█▏        | 70/567 [07:01<53:23,  6.45s/it]
 13%|█▎        | 71/567 [07:04<45:07,  5.46s/it]
 13%|█▎        | 72/567 [07:08<42:46,  5.18s/it]
 13%|█▎        | 73/567 [07:12<38:00,  4.62s/it]
 13%|█▎        | 74/567 [07:17<39:01,  4.75s/it]
 13%|█▎        | 75/567 [07:20<35:24,  4.32s/it]
 13%|█▎        | 76/567 [07:25<37:29,  4.58s/it]
 14%|█▎        | 77/567 [07:34<47:42,  5.84s/it]
 14%|█▍        | 78/567 [07:39<45:00,  5.52s/it]
 14%|█▍        | 79/567 [07:44<43:23,  5.34s/it]
 14%|█▍        | 80/567 [07:47<38:42,  4.77s/it]
 14%|█▍        | 81/567 [07:51<35:46,  4.42s/it]
 14%|█▍        | 82/567 [07:55<34:22,  4.25s/it]
 15%|█▍        | 83/567 [07:59<33:44,  4.18s/it]
 15%|█▍        | 84/567 [08:02<31:33,  3.92s/it]
 15%|█▍        | 85/567 [08:04<25:50,  3.22s/it]
 15%|█▌        | 86/567 [08:05<22:08,  2.76s/it]
 15%|█▌        | 87/567 [08:08<21:37,  2.70s/it]
 16%|█▌        | 88/567 [08:12<24:32,  3.07s/it]
 16%|█▌        | 89/567 [08:25<47:53,  6.01s/it]
 16%|█▌        | 90/567 [08:30<46:54,  5.90s/it]
 16%|█▌        | 91/567 [08:37<48:38,  6.13s/it]
 16%|█▌        | 92/567 [08:39<39:05,  4.94s/it]
 16%|█▋        | 93/567 [08:41<32:42,  4.14s/it]
 17%|█▋        | 94/567 [08:50<42:38,  5.41s/it]
 17%|█▋        | 95/567 [08:53<36:32,  4.64s/it]
 17%|█▋        | 96/567 [08:58<37:33,  4.78s/it]
 17%|█▋        | 97/567 [09:01<33:35,  4.29s/it]
 17%|█▋        | 98/567 [09:03<28:49,  3.69s/it]
 17%|█▋        | 99/567 [09:07<28:44,  3.68s/it]
 18%|█▊        | 100/567 [09:15<39:54,  5.13s/it]
 18%|█▊        | 101/567 [09:19<36:47,  4.74s/it]
 18%|█▊        | 102/567 [09:28<47:02,  6.07s/it]
 18%|█▊        | 103/567 [09:30<36:09,  4.68s/it]
 18%|█▊        | 104/567 [09:38<43:25,  5.63s/it]
 19%|█▊        | 105/567 [09:51<1:01:29,  7.99s/it]
 19%|█▊        | 106/567 [09:58<1:00:02,  7.81s/it]
 19%|█▉        | 107/567 [10:01<48:50,  6.37s/it]  
 19%|█▉        | 108/567 [10:17<1:10:13,  9.18s/it]
 19%|█▉        | 109/567 [10:19<52:19,  6.86s/it]  
 19%|█▉        | 110/567 [10:22<44:27,  5.84s/it]
 20%|█▉        | 111/567 [10:36<1:03:31,  8.36s/it]
 20%|█▉        | 112/567 [10:44<1:01:33,  8.12s/it]
 20%|█▉        | 113/567 [10:56<1:10:17,  9.29s/it]
 20%|██        | 114/567 [11:00<58:37,  7.76s/it]  
 20%|██        | 115/567 [11:05<51:37,  6.85s/it]
 20%|██        | 116/567 [11:08<42:56,  5.71s/it]
 21%|██        | 117/567 [11:12<38:39,  5.15s/it]
 21%|██        | 118/567 [11:14<32:54,  4.40s/it]
 21%|██        | 119/567 [11:25<47:30,  6.36s/it]
 21%|██        | 120/567 [11:29<41:38,  5.59s/it]
 21%|██▏       | 121/567 [11:32<34:46,  4.68s/it]
 22%|██▏       | 122/567 [11:35<31:03,  4.19s/it]
 22%|██▏       | 123/567 [11:39<31:11,  4.21s/it]
 22%|██▏       | 124/567 [11:42<27:49,  3.77s/it]
 22%|██▏       | 125/567 [11:45<26:43,  3.63s/it]
 22%|██▏       | 126/567 [11:50<29:24,  4.00s/it]
 22%|██▏       | 127/567 [12:00<42:27,  5.79s/it]
 23%|██▎       | 128/567 [12:07<45:49,  6.26s/it]
 23%|██▎       | 129/567 [12:16<50:10,  6.87s/it]
 23%|██▎       | 130/567 [12:27<1:00:37,  8.32s/it]
 23%|██▎       | 131/567 [12:41<1:12:29,  9.98s/it]
 23%|██▎       | 132/567 [12:53<1:15:47, 10.45s/it]
 23%|██▎       | 133/567 [13:10<1:29:51, 12.42s/it]
 24%|██▎       | 134/567 [13:30<1:46:29, 14.76s/it]
 24%|██▍       | 135/567 [13:35<1:25:48, 11.92s/it]
 24%|██▍       | 136/567 [13:41<1:12:25, 10.08s/it]
 24%|██▍       | 137/567 [13:44<57:07,  7.97s/it]  
 24%|██▍       | 138/567 [13:47<46:02,  6.44s/it]
 25%|██▍       | 139/567 [13:51<40:58,  5.75s/it]
 25%|██▍       | 140/567 [13:56<38:39,  5.43s/it]
 25%|██▍       | 141/567 [14:01<38:48,  5.47s/it]
 25%|██▌       | 142/567 [14:04<33:12,  4.69s/it]
 25%|██▌       | 143/567 [14:07<29:18,  4.15s/it]
 25%|██▌       | 144/567 [14:12<31:41,  4.50s/it]
 26%|██▌       | 145/567 [14:18<34:24,  4.89s/it]
 26%|██▌       | 146/567 [14:23<35:09,  5.01s/it]
 26%|██▌       | 147/567 [14:39<58:11,  8.31s/it]
 26%|██▌       | 148/567 [14:45<51:40,  7.40s/it]
 26%|██▋       | 149/567 [14:47<41:45,  5.99s/it]
 26%|██▋       | 150/567 [14:49<31:50,  4.58s/it]
 27%|██▋       | 151/567 [14:50<25:29,  3.68s/it]
 27%|██▋       | 152/567 [14:55<28:11,  4.08s/it]
 27%|██▋       | 153/567 [15:01<32:22,  4.69s/it]
 27%|██▋       | 154/567 [15:04<28:48,  4.19s/it]
 27%|██▋       | 155/567 [15:10<32:11,  4.69s/it]
 28%|██▊       | 156/567 [15:14<30:31,  4.46s/it]
 28%|██▊       | 157/567 [15:18<29:26,  4.31s/it]
 28%|██▊       | 158/567 [15:30<44:04,  6.47s/it]
 28%|██▊       | 159/567 [15:34<38:48,  5.71s/it]
 28%|██▊       | 160/567 [15:44<47:50,  7.05s/it]
 28%|██▊       | 161/567 [15:46<37:10,  5.49s/it]
 29%|██▊       | 162/567 [15:52<39:13,  5.81s/it]
 29%|██▊       | 163/567 [15:55<34:03,  5.06s/it]
 29%|██▉       | 164/567 [17:01<2:36:28, 23.30s/it]
 29%|██▉       | 165/567 [17:07<2:01:37, 18.15s/it]
 29%|██▉       | 166/567 [17:11<1:32:58, 13.91s/it]
 29%|██▉       | 167/567 [17:22<1:26:07, 12.92s/it]
 30%|██▉       | 168/567 [17:26<1:07:29, 10.15s/it]
 30%|██▉       | 169/567 [17:32<58:59,  8.89s/it]  
 30%|██▉       | 170/567 [17:39<54:48,  8.28s/it]
 30%|███       | 171/567 [17:45<50:33,  7.66s/it]
 30%|███       | 172/567 [17:51<47:54,  7.28s/it]
 31%|███       | 173/567 [17:55<40:29,  6.17s/it]
 31%|███       | 174/567 [18:00<38:22,  5.86s/it]
 31%|███       | 175/567 [18:06<38:41,  5.92s/it]
 31%|███       | 176/567 [18:10<34:29,  5.29s/it]
 31%|███       | 177/567 [18:20<43:13,  6.65s/it]
 31%|███▏      | 178/567 [18:23<36:19,  5.60s/it]
 32%|███▏      | 179/567 [18:28<36:10,  5.59s/it]
 32%|███▏      | 180/567 [18:34<35:20,  5.48s/it]
 32%|███▏      | 181/567 [18:39<35:17,  5.49s/it]
 32%|███▏      | 182/567 [18:41<27:55,  4.35s/it]
 32%|███▏      | 183/567 [18:43<23:05,  3.61s/it]
 32%|███▏      | 184/567 [18:46<22:46,  3.57s/it]
 33%|███▎      | 185/567 [18:50<23:17,  3.66s/it]
 33%|███▎      | 186/567 [19:12<58:10,  9.16s/it]
 33%|███▎      | 187/567 [19:16<49:06,  7.75s/it]
 33%|███▎      | 188/567 [19:19<39:25,  6.24s/it]
 33%|███▎      | 189/567 [19:24<37:11,  5.90s/it]
 34%|███▎      | 190/567 [19:27<31:21,  4.99s/it]
 34%|███▎      | 191/567 [19:36<39:18,  6.27s/it]
 34%|███▍      | 192/567 [19:43<39:42,  6.35s/it]
 34%|███▍      | 193/567 [20:01<1:00:36,  9.72s/it]
 34%|███▍      | 194/567 [20:04<47:54,  7.71s/it]  
 34%|███▍      | 195/567 [20:08<41:46,  6.74s/it]
 35%|███▍      | 196/567 [20:13<38:14,  6.19s/it]
 35%|███▍      | 197/567 [20:24<46:35,  7.56s/it]
 35%|███▍      | 198/567 [20:33<49:08,  7.99s/it]
 35%|███▌      | 199/567 [20:38<44:22,  7.23s/it]
 35%|███▌      | 200/567 [20:42<38:49,  6.35s/it]
 35%|███▌      | 201/567 [20:45<31:48,  5.21s/it]
 36%|███▌      | 202/567 [20:49<29:20,  4.82s/it]
 36%|███▌      | 203/567 [20:52<26:13,  4.32s/it]
 36%|███▌      | 204/567 [20:55<23:45,  3.93s/it]
 36%|███▌      | 205/567 [21:00<25:57,  4.30s/it]
 36%|███▋      | 206/567 [21:11<37:26,  6.22s/it]
 37%|███▋      | 207/567 [21:16<36:05,  6.02s/it]
 37%|███▋      | 208/567 [21:25<39:39,  6.63s/it]
 37%|███▋      | 209/567 [21:26<30:44,  5.15s/it]
 37%|███▋      | 210/567 [21:33<33:56,  5.70s/it]
 37%|███▋      | 211/567 [21:37<31:00,  5.23s/it]
 37%|███▋      | 212/567 [21:42<29:59,  5.07s/it]
 38%|███▊      | 213/567 [21:48<31:09,  5.28s/it]
 38%|███▊      | 214/567 [21:54<31:54,  5.42s/it]
 38%|███▊      | 215/567 [22:00<34:07,  5.82s/it]
 38%|███▊      | 216/567 [22:08<36:59,  6.32s/it]
 38%|███▊      | 217/567 [22:15<38:30,  6.60s/it]
 38%|███▊      | 218/567 [22:23<40:27,  6.96s/it]
 39%|███▊      | 219/567 [22:29<38:33,  6.65s/it]
 39%|███▉      | 220/567 [22:32<33:10,  5.74s/it]
 39%|███▉      | 221/567 [22:35<27:06,  4.70s/it]
 39%|███▉      | 222/567 [22:38<25:30,  4.44s/it]
 39%|███▉      | 223/567 [22:42<23:28,  4.10s/it]
 40%|███▉      | 224/567 [22:46<23:34,  4.12s/it]
 40%|███▉      | 225/567 [22:56<33:40,  5.91s/it]
 40%|███▉      | 226/567 [23:02<34:04,  6.00s/it]
 40%|████      | 227/567 [23:04<27:11,  4.80s/it]
 40%|████      | 228/567 [23:09<27:18,  4.83s/it]
 40%|████      | 229/567 [23:15<29:34,  5.25s/it]
 41%|████      | 230/567 [23:18<25:44,  4.58s/it]
 41%|████      | 231/567 [23:27<32:29,  5.80s/it]
 41%|████      | 232/567 [23:32<31:11,  5.59s/it]
 41%|████      | 233/567 [23:41<37:19,  6.71s/it]
 41%|████▏     | 234/567 [23:44<30:05,  5.42s/it]
 41%|████▏     | 235/567 [23:48<28:41,  5.18s/it]
 42%|████▏     | 236/567 [23:54<29:25,  5.33s/it]
 42%|████▏     | 237/567 [23:58<27:11,  4.94s/it]
 42%|████▏     | 238/567 [24:03<27:15,  4.97s/it]
 42%|████▏     | 239/567 [24:08<26:57,  4.93s/it]
 42%|████▏     | 240/567 [24:20<39:01,  7.16s/it]
 43%|████▎     | 241/567 [24:26<35:32,  6.54s/it]
 43%|████▎     | 242/567 [24:29<30:50,  5.69s/it]
 43%|████▎     | 243/567 [24:37<34:11,  6.33s/it]
 43%|████▎     | 244/567 [24:45<37:08,  6.90s/it]
 43%|████▎     | 245/567 [24:48<29:49,  5.56s/it]
 43%|████▎     | 246/567 [24:53<29:16,  5.47s/it]
 44%|████▎     | 247/567 [24:55<24:04,  4.51s/it]
 44%|████▎     | 248/567 [25:00<23:56,  4.50s/it]
 44%|████▍     | 249/567 [25:04<22:47,  4.30s/it]
 44%|████▍     | 250/567 [25:13<30:58,  5.86s/it]
 44%|████▍     | 251/567 [25:24<38:32,  7.32s/it]
 44%|████▍     | 252/567 [25:31<38:31,  7.34s/it]
 45%|████▍     | 253/567 [25:34<30:41,  5.86s/it]
 45%|████▍     | 254/567 [25:46<40:09,  7.70s/it]
 45%|████▍     | 255/567 [25:52<37:40,  7.25s/it]
 45%|████▌     | 256/567 [25:55<31:11,  6.02s/it]
 45%|████▌     | 257/567 [26:09<43:43,  8.46s/it]
 46%|████▌     | 258/567 [26:15<39:38,  7.70s/it]
 46%|████▌     | 259/567 [26:18<32:45,  6.38s/it]
 46%|████▌     | 260/567 [26:34<47:14,  9.23s/it]
 46%|████▌     | 261/567 [26:41<42:51,  8.40s/it]
 46%|████▌     | 262/567 [26:46<38:13,  7.52s/it]
 46%|████▋     | 263/567 [26:49<31:40,  6.25s/it]
 47%|████▋     | 264/567 [26:51<24:42,  4.89s/it]
 47%|████▋     | 265/567 [27:09<44:51,  8.91s/it]
 47%|████▋     | 266/567 [27:12<35:37,  7.10s/it]
 47%|████▋     | 267/567 [27:14<27:51,  5.57s/it]
 47%|████▋     | 268/567 [27:24<34:20,  6.89s/it]
 47%|████▋     | 269/567 [27:36<41:31,  8.36s/it]
 48%|████▊     | 270/567 [27:41<36:04,  7.29s/it]
 48%|████▊     | 271/567 [27:51<40:08,  8.14s/it]
 48%|████▊     | 272/567 [27:55<33:28,  6.81s/it]
 48%|████▊     | 273/567 [27:58<28:34,  5.83s/it]
 48%|████▊     | 274/567 [28:01<24:34,  5.03s/it]
 49%|████▊     | 275/567 [28:05<22:55,  4.71s/it]
 49%|████▊     | 276/567 [28:10<22:55,  4.73s/it]
 49%|████▉     | 277/567 [28:17<26:36,  5.51s/it]
 49%|████▉     | 278/567 [28:19<21:26,  4.45s/it]
 49%|████▉     | 279/567 [28:33<34:24,  7.17s/it]
 49%|████▉     | 280/567 [28:40<33:43,  7.05s/it]
 50%|████▉     | 281/567 [28:42<27:11,  5.71s/it]
 50%|████▉     | 282/567 [28:44<22:02,  4.64s/it]
 50%|████▉     | 283/567 [28:56<31:54,  6.74s/it]
 50%|█████     | 284/567 [29:00<27:19,  5.79s/it]
 50%|█████     | 285/567 [29:04<25:14,  5.37s/it]
 50%|█████     | 286/567 [29:07<21:37,  4.62s/it]
 51%|█████     | 287/567 [29:15<26:43,  5.73s/it]
 51%|█████     | 288/567 [29:22<28:37,  6.15s/it]
 51%|█████     | 289/567 [29:30<30:06,  6.50s/it]
 51%|█████     | 290/567 [29:35<28:55,  6.27s/it]
 51%|█████▏    | 291/567 [29:43<30:42,  6.68s/it]
 51%|█████▏    | 292/567 [29:51<32:15,  7.04s/it]
 52%|█████▏    | 293/567 [29:56<29:07,  6.38s/it]
 52%|█████▏    | 294/567 [29:59<25:20,  5.57s/it]
 52%|█████▏    | 295/567 [30:03<22:23,  4.94s/it]
 52%|█████▏    | 296/567 [30:09<23:26,  5.19s/it]
 52%|█████▏    | 297/567 [30:12<20:13,  4.49s/it]
 53%|█████▎    | 298/567 [30:27<34:27,  7.68s/it]
 53%|█████▎    | 299/567 [30:31<29:46,  6.67s/it]
 53%|█████▎    | 300/567 [30:37<28:56,  6.50s/it]
 53%|█████▎    | 301/567 [30:43<27:25,  6.18s/it]
 53%|█████▎    | 302/567 [30:48<26:05,  5.91s/it]
 53%|█████▎    | 303/567 [30:54<25:46,  5.86s/it]
 54%|█████▎    | 304/567 [30:57<22:58,  5.24s/it]
 54%|█████▍    | 305/567 [31:00<19:38,  4.50s/it]
 54%|█████▍    | 306/567 [31:08<24:13,  5.57s/it]
 54%|█████▍    | 307/567 [31:19<30:22,  7.01s/it]
 54%|█████▍    | 308/567 [31:21<24:33,  5.69s/it]
 54%|█████▍    | 309/567 [31:23<19:41,  4.58s/it]
 55%|█████▍    | 310/567 [31:38<33:16,  7.77s/it]
 55%|█████▍    | 311/567 [31:47<34:19,  8.05s/it]
 55%|█████▌    | 312/567 [31:51<29:01,  6.83s/it]
 55%|█████▌    | 313/567 [31:55<24:59,  5.90s/it]
 55%|█████▌    | 314/567 [32:02<26:32,  6.30s/it]
 56%|█████▌    | 315/567 [32:15<34:54,  8.31s/it]
 56%|█████▌    | 316/567 [32:17<27:12,  6.51s/it]
 56%|█████▌    | 317/567 [32:22<24:30,  5.88s/it]
 56%|█████▌    | 318/567 [32:24<19:45,  4.76s/it]
 56%|█████▋    | 319/567 [32:31<22:55,  5.55s/it]
 56%|█████▋    | 320/567 [32:33<17:55,  4.35s/it]
 57%|█████▋    | 321/567 [32:47<29:41,  7.24s/it]
 57%|█████▋    | 322/567 [32:53<28:20,  6.94s/it]
 57%|█████▋    | 323/567 [32:58<26:16,  6.46s/it]
 57%|█████▋    | 324/567 [33:01<21:50,  5.39s/it]
 57%|█████▋    | 325/567 [33:16<33:24,  8.28s/it]
 57%|█████▋    | 326/567 [33:18<25:41,  6.40s/it]
 58%|█████▊    | 327/567 [33:22<21:53,  5.47s/it]
 58%|█████▊    | 328/567 [33:28<22:48,  5.73s/it]
 58%|█████▊    | 329/567 [33:38<27:32,  6.94s/it]
 58%|█████▊    | 330/567 [33:39<21:03,  5.33s/it]
 58%|█████▊    | 331/567 [33:43<19:01,  4.84s/it]
 59%|█████▊    | 332/567 [33:47<17:43,  4.53s/it]
 59%|█████▊    | 333/567 [33:51<17:37,  4.52s/it]
 59%|█████▉    | 334/567 [33:55<16:24,  4.23s/it]
 59%|█████▉    | 335/567 [33:57<14:17,  3.70s/it]
 59%|█████▉    | 336/567 [34:00<12:35,  3.27s/it]
 59%|█████▉    | 337/567 [34:07<16:43,  4.36s/it]
 60%|█████▉    | 338/567 [34:10<15:26,  4.05s/it]
 60%|█████▉    | 339/567 [34:13<13:51,  3.65s/it]
 60%|█████▉    | 340/567 [34:16<13:14,  3.50s/it]
 60%|██████    | 341/567 [34:18<12:08,  3.22s/it]
 60%|██████    | 342/567 [34:24<14:26,  3.85s/it]
 60%|██████    | 343/567 [34:26<12:38,  3.38s/it]
 61%|██████    | 344/567 [34:30<13:28,  3.63s/it]
 61%|██████    | 345/567 [34:34<14:10,  3.83s/it]
 61%|██████    | 346/567 [34:38<14:06,  3.83s/it]
 61%|██████    | 347/567 [34:43<14:53,  4.06s/it]
 61%|██████▏   | 348/567 [34:47<15:07,  4.14s/it]
 62%|██████▏   | 349/567 [34:56<20:10,  5.55s/it]
 62%|██████▏   | 350/567 [35:01<18:58,  5.25s/it]
 62%|██████▏   | 351/567 [35:21<35:41,  9.91s/it]
 62%|██████▏   | 352/567 [35:25<28:46,  8.03s/it]
 62%|██████▏   | 353/567 [35:36<31:45,  8.90s/it]
 62%|██████▏   | 354/567 [35:42<28:47,  8.11s/it]
 63%|██████▎   | 355/567 [35:47<24:54,  7.05s/it]
 63%|██████▎   | 356/567 [35:52<23:09,  6.58s/it]
 63%|██████▎   | 357/567 [35:54<18:22,  5.25s/it]
 63%|██████▎   | 358/567 [35:58<16:53,  4.85s/it]
 63%|████srun: Job step aborted: Waiting up to 17 seconds for job step to finish.
+██▎   | 359/567 [36:01<15:03,  4.34s/it]slurmstepd: error: *** STEP 76696.0 ON lagoon CANCELLED AT 2024-08-09T23:06:32 ***
+slurmstepd: error: *** JOB 76696 ON lagoon CANCELLED AT 2024-08-09T23:06:32 ***
+Job ID: 76696
+Cluster: crimson
+User/Group: dh.huang.2023/dh.huang.2023
+State: CANCELLED (exit code 0)
+Nodes: 1
+Cores per node: 10
+CPU Utilized: 00:00:02
+CPU Efficiency: 0.00% of 2-13:23:40 core-walltime
+Job Wall-clock time: 06:08:22
+Memory Utilized: 2.32 GB
+Memory Efficiency: 0.91% of 256.00 GB

results/mac-results_fine_tuned.csv

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:3157c0a6bc7d611450e81cb6b9537da3b0a67cb29e45a254acd107658b2bc712
-size 4194027
+oid sha256:32593c2a7be738a20005c4cee5a101b551b60d460c68cdd4fca4e5789257b982
+size 4724172

scripts/eval-mac.sh

@@ -32,6 +32,8 @@ grep MemTotal /proc/meminfo
 
 # ./scripts/eval-epochs.sh shenzhi-wang Llama3.1-8B-Chinese-Chat
 
+export END_REPETITION_PENALTY=1.0
+
 ./scripts/eval-rpp.sh internlm internlm2_5-7b-chat checkpoint-140
 
 ./scripts/eval-rpp.sh Qwen Qwen2-7B-Instruct checkpoint-105

scripts/tune-mac-4gpu.sh

@@ -18,6 +18,8 @@ grep MemTotal /proc/meminfo
 
 #./scripts/tune-lf-4gpu.sh shenzhi-wang Llama3.1-70B-Chinese-Chat llama3
 
+export LOAD_IN_4BIT=true
+
 ./scripts/eval-epochs.sh Qwen Qwen2-72B-Instruct
 
 ./scripts/eval-epochs.sh shenzhi-wang Llama3.1-70B-Chinese-Chat