Model save

README.md

@@ -26,7 +26,7 @@ print(output["generated_text"])
 
 ## Training procedure
 
-[<img src="https://raw.githubusercontent.com/wandb/assets/main/wandb-github-badge-28.svg" alt="Visualize in Weights & Biases" width="150" height="24"/>](https://wandb.ai/ggbetz/argunauts-training/runs/4qlrj3fp) 
+[<img src="https://raw.githubusercontent.com/wandb/assets/main/wandb-github-badge-28.svg" alt="Visualize in Weights & Biases" width="150" height="24"/>](https://wandb.ai/ggbetz/argunauts-training/runs/d55470xf) 
 
 
 This model was trained with DPO, a method introduced in [Direct Preference Optimization: Your Language Model is Secretly a Reward Model](https://huggingface.co/papers/2305.18290).

all_results.json

@@ -1,9 +1,9 @@
 {
-    "epoch": 1.996219281663516,
+    "epoch": 1.9971305595408895,
     "total_flos": 0.0,
-    "train_loss": 0.5393935610549618,
-    "train_runtime": 2398.307,
-    "train_samples": 6347,
-    "train_samples_per_second": 5.293,
-    "train_steps_per_second": 0.083
+    "train_loss": 0.5929731760901966,
+    "train_runtime": 2030.2211,
+    "train_samples": 5576,
+    "train_samples_per_second": 5.493,
+    "train_steps_per_second": 0.086
 }

model-00001-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:8f807a55400211dc38fe9871d39bbdb3a1c49f8b79532d4313e1c899fe429e10
+oid sha256:f1b2cfa01d7837330c9b890a79677233bde135efb7b9300dc70ca9c2436cfe2d
 size 4976698672

model-00002-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b8dd55c15ea1762fc4462e86e4d79d32fb9df2bf76c79e3c0e53edca6e2575d8
+oid sha256:f1a9f875c8b82e62970bc18ae27be339c4fa9058b0cdb3f537fa10774bf479e9
 size 4999802720

model-00003-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:954125f0f6d5961aff8db190d00a9831a874a385c04dedda7d82359384d7233e
+oid sha256:f23e9719d839b39d06c0e5b2276ea0def1e8e6d6774413af2ae74d34fa9ac0a6
 size 4915916176

model-00004-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a4c9947d0ccbc72142d87c4ef362037fd555ee53092f6642a644678e9be9d61e
+oid sha256:643861ba758160b88b3f43351ea8d2c440cf58ff16173e0337f35c74c90cf95b
 size 1168138808

train_results.json

@@ -1,9 +1,9 @@
 {
-    "epoch": 1.996219281663516,
+    "epoch": 1.9971305595408895,
     "total_flos": 0.0,
-    "train_loss": 0.5393935610549618,
-    "train_runtime": 2398.307,
-    "train_samples": 6347,
-    "train_samples_per_second": 5.293,
-    "train_steps_per_second": 0.083
+    "train_loss": 0.5929731760901966,
+    "train_runtime": 2030.2211,
+    "train_samples": 5576,
+    "train_samples_per_second": 5.493,
+    "train_steps_per_second": 0.086
 }

trainer_state.json

@@ -1,610 +1,535 @@
 {
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 1.996219281663516,
+  "epoch": 1.9971305595408895,
   "eval_steps": 500,
-  "global_step": 198,
+  "global_step": 174,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
   "log_history": [
     {
-      "epoch": 0.05040957781978576,
-      "grad_norm": 108.41279315573027,
-      "learning_rate": 1e-07,
+      "epoch": 0.05738880918220947,
+      "grad_norm": 147.0504219778771,
+      "learning_rate": 1.1111111111111111e-07,
       "logits/chosen": NaN,
       "logits/rejected": NaN,
-      "logps/chosen": -394.29998779296875,
-      "logps/rejected": -490.4984436035156,
-      "loss": 0.6973,
-      "rewards/accuracies": 0.12812499701976776,
-      "rewards/chosen": -0.0023020743392407894,
-      "rewards/margins": -0.009827613830566406,
-      "rewards/rejected": 0.007504081819206476,
+      "logps/chosen": -309.26251220703125,
+      "logps/rejected": -410.8433532714844,
+      "loss": 0.6899,
+      "rewards/accuracies": 0.17499999701976776,
+      "rewards/chosen": -0.002740192459896207,
+      "rewards/margins": 0.0071624754928052425,
+      "rewards/rejected": -0.00990285910665989,
       "step": 5
     },
     {
-      "epoch": 0.10081915563957151,
-      "grad_norm": 87.2429177419689,
-      "learning_rate": 2e-07,
+      "epoch": 0.11477761836441894,
+      "grad_norm": 123.58633090508155,
+      "learning_rate": 1.9878787878787876e-07,
       "logits/chosen": NaN,
       "logits/rejected": NaN,
-      "logps/chosen": -374.5687561035156,
-      "logps/rejected": -397.4437561035156,
-      "loss": 0.6827,
-      "rewards/accuracies": 0.30937498807907104,
-      "rewards/chosen": 0.009921550750732422,
-      "rewards/margins": 0.029467200860381126,
-      "rewards/rejected": -0.01954820193350315,
+      "logps/chosen": -277.09844970703125,
+      "logps/rejected": -374.31719970703125,
+      "loss": 0.6926,
+      "rewards/accuracies": 0.2718749940395355,
+      "rewards/chosen": 0.0009648323175497353,
+      "rewards/margins": 0.00359344482421875,
+      "rewards/rejected": -0.0026039122603833675,
       "step": 10
     },
     {
-      "epoch": 0.15122873345935728,
-      "grad_norm": 83.76505879098643,
-      "learning_rate": 1.946808510638298e-07,
+      "epoch": 0.17216642754662842,
+      "grad_norm": 82.52640749761395,
+      "learning_rate": 1.9272727272727272e-07,
       "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": -361.41485595703125,
-      "logps/rejected": -441.2671813964844,
-      "loss": 0.6832,
-      "rewards/accuracies": 0.3187499940395355,
-      "rewards/chosen": -0.0191789623349905,
-      "rewards/margins": 0.04044074937701225,
-      "rewards/rejected": -0.059579335153102875,
+      "logits/rejected": -0.355844110250473,
+      "logps/chosen": -250.90859985351562,
+      "logps/rejected": -288.7484436035156,
+      "loss": 0.6679,
+      "rewards/accuracies": 0.3656249940395355,
+      "rewards/chosen": -0.018105220049619675,
+      "rewards/margins": 0.07349129021167755,
+      "rewards/rejected": -0.09159164130687714,
       "step": 15
     },
     {
-      "epoch": 0.20163831127914303,
-      "grad_norm": 55.583186387674,
-      "learning_rate": 1.8936170212765957e-07,
+      "epoch": 0.22955523672883787,
+      "grad_norm": 46.811267261200435,
+      "learning_rate": 1.8666666666666667e-07,
       "logits/chosen": NaN,
       "logits/rejected": NaN,
-      "logps/chosen": -373.5375061035156,
-      "logps/rejected": -464.6875,
-      "loss": 0.6233,
-      "rewards/accuracies": 0.4437499940395355,
-      "rewards/chosen": -0.021530818194150925,
-      "rewards/margins": 0.23837146162986755,
-      "rewards/rejected": -0.2597528398036957,
+      "logps/chosen": -250.43203735351562,
+      "logps/rejected": -299.9468688964844,
+      "loss": 0.638,
+      "rewards/accuracies": 0.4124999940395355,
+      "rewards/chosen": -0.02562398836016655,
+      "rewards/margins": 0.21783074736595154,
+      "rewards/rejected": -0.24351105093955994,
       "step": 20
     },
     {
-      "epoch": 0.2520478890989288,
-      "grad_norm": 60.717924564454115,
-      "learning_rate": 1.8404255319148937e-07,
+      "epoch": 0.28694404591104733,
+      "grad_norm": 44.812641031283796,
+      "learning_rate": 1.806060606060606e-07,
       "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": -371.62188720703125,
-      "logps/rejected": -478.8374938964844,
-      "loss": 0.6328,
-      "rewards/accuracies": 0.3968749940395355,
-      "rewards/chosen": -0.04529209062457085,
-      "rewards/margins": 0.28455600142478943,
-      "rewards/rejected": -0.3299552798271179,
+      "logits/rejected": -0.32989805936813354,
+      "logps/chosen": -264.3968811035156,
+      "logps/rejected": -420.49688720703125,
+      "loss": 0.6218,
+      "rewards/accuracies": 0.4625000059604645,
+      "rewards/chosen": -0.05606970936059952,
+      "rewards/margins": 0.3526493012905121,
+      "rewards/rejected": -0.4085969924926758,
       "step": 25
     },
     {
-      "epoch": 0.30245746691871456,
-      "grad_norm": 58.667280799471335,
-      "learning_rate": 1.7872340425531914e-07,
+      "epoch": 0.34433285509325684,
+      "grad_norm": 41.09665846804977,
+      "learning_rate": 1.7454545454545453e-07,
       "logits/chosen": NaN,
       "logits/rejected": NaN,
-      "logps/chosen": -407.125,
-      "logps/rejected": -456.94061279296875,
-      "loss": 0.6037,
-      "rewards/accuracies": 0.4281249940395355,
-      "rewards/chosen": -0.06888346374034882,
-      "rewards/margins": 0.4735303819179535,
-      "rewards/rejected": -0.5425974726676941,
+      "logps/chosen": -261.12969970703125,
+      "logps/rejected": -333.25665283203125,
+      "loss": 0.6433,
+      "rewards/accuracies": 0.453125,
+      "rewards/chosen": -0.05041093751788139,
+      "rewards/margins": 0.3066027760505676,
+      "rewards/rejected": -0.3569931983947754,
       "step": 30
     },
     {
-      "epoch": 0.35286704473850034,
-      "grad_norm": 52.677451346587404,
-      "learning_rate": 1.7340425531914892e-07,
+      "epoch": 0.4017216642754663,
+      "grad_norm": 47.66979279242685,
+      "learning_rate": 1.6848484848484848e-07,
       "logits/chosen": NaN,
       "logits/rejected": NaN,
-      "logps/chosen": -396.4683532714844,
-      "logps/rejected": -433.375,
-      "loss": 0.5971,
-      "rewards/accuracies": 0.515625,
-      "rewards/chosen": -0.10200033336877823,
-      "rewards/margins": 0.5540359616279602,
-      "rewards/rejected": -0.6558942794799805,
+      "logps/chosen": -252.76406860351562,
+      "logps/rejected": -320.32342529296875,
+      "loss": 0.611,
+      "rewards/accuracies": 0.546875,
+      "rewards/chosen": -0.07408180087804794,
+      "rewards/margins": 0.735063910484314,
+      "rewards/rejected": -0.8097826838493347,
       "step": 35
     },
     {
-      "epoch": 0.40327662255828606,
-      "grad_norm": 61.023691806589966,
-      "learning_rate": 1.6808510638297872e-07,
+      "epoch": 0.45911047345767575,
+      "grad_norm": 44.453842871184655,
+      "learning_rate": 1.624242424242424e-07,
       "logits/chosen": NaN,
       "logits/rejected": NaN,
-      "logps/chosen": -349.65545654296875,
-      "logps/rejected": -466.68438720703125,
-      "loss": 0.6065,
-      "rewards/accuracies": 0.578125,
-      "rewards/chosen": -0.10066480934619904,
-      "rewards/margins": 0.6059595346450806,
-      "rewards/rejected": -0.7062518000602722,
+      "logps/chosen": -282.52734375,
+      "logps/rejected": -314.4585876464844,
+      "loss": 0.6154,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -0.08205080032348633,
+      "rewards/margins": 0.6658231616020203,
+      "rewards/rejected": -0.7472448348999023,
       "step": 40
     },
     {
-      "epoch": 0.45368620037807184,
-      "grad_norm": 50.9687221759888,
-      "learning_rate": 1.627659574468085e-07,
+      "epoch": 0.5164992826398852,
+      "grad_norm": 72.40873860390725,
+      "learning_rate": 1.5636363636363637e-07,
       "logits/chosen": NaN,
-      "logits/rejected": -0.1692344695329666,
-      "logps/chosen": -325.9609375,
-      "logps/rejected": -462.97186279296875,
-      "loss": 0.5598,
-      "rewards/accuracies": 0.6031249761581421,
-      "rewards/chosen": -0.09121231734752655,
-      "rewards/margins": 1.08050537109375,
-      "rewards/rejected": -1.1715847253799438,
+      "logits/rejected": NaN,
+      "logps/chosen": -246.61483764648438,
+      "logps/rejected": -324.8515625,
+      "loss": 0.6541,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -0.14140835404396057,
+      "rewards/margins": 0.6368468999862671,
+      "rewards/rejected": -0.7777351140975952,
       "step": 45
     },
     {
-      "epoch": 0.5040957781978576,
-      "grad_norm": 56.54625986325724,
-      "learning_rate": 1.574468085106383e-07,
+      "epoch": 0.5738880918220947,
+      "grad_norm": 51.33061405888651,
+      "learning_rate": 1.503030303030303e-07,
       "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": -342.57501220703125,
-      "logps/rejected": -478.74139404296875,
-      "loss": 0.591,
-      "rewards/accuracies": 0.574999988079071,
-      "rewards/chosen": -0.10402297973632812,
-      "rewards/margins": 0.7732677459716797,
-      "rewards/rejected": -0.8772258758544922,
+      "logits/rejected": -0.29754638671875,
+      "logps/chosen": -251.82656860351562,
+      "logps/rejected": -364.16094970703125,
+      "loss": 0.6061,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -0.097315214574337,
+      "rewards/margins": 0.7471939325332642,
+      "rewards/rejected": -0.8438205718994141,
       "step": 50
     },
     {
-      "epoch": 0.5545053560176434,
-      "grad_norm": 52.007398326371984,
-      "learning_rate": 1.5212765957446807e-07,
+      "epoch": 0.6312769010043041,
+      "grad_norm": 47.78546749711004,
+      "learning_rate": 1.4424242424242422e-07,
       "logits/chosen": NaN,
       "logits/rejected": NaN,
-      "logps/chosen": -381.3374938964844,
-      "logps/rejected": -456.14373779296875,
-      "loss": 0.5847,
-      "rewards/accuracies": 0.574999988079071,
-      "rewards/chosen": -0.08334217220544815,
-      "rewards/margins": 0.8058792352676392,
-      "rewards/rejected": -0.889452338218689,
+      "logps/chosen": -229.6281280517578,
+      "logps/rejected": -346.20623779296875,
+      "loss": 0.5885,
+      "rewards/accuracies": 0.596875011920929,
+      "rewards/chosen": -0.08479080349206924,
+      "rewards/margins": 0.9593955874443054,
+      "rewards/rejected": -1.0437196493148804,
       "step": 55
     },
     {
-      "epoch": 0.6049149338374291,
-      "grad_norm": 46.410753552087435,
-      "learning_rate": 1.4680851063829787e-07,
+      "epoch": 0.6886657101865137,
+      "grad_norm": 32.88856633193689,
+      "learning_rate": 1.3818181818181818e-07,
       "logits/chosen": NaN,
       "logits/rejected": NaN,
-      "logps/chosen": -358.54998779296875,
-      "logps/rejected": -475.54998779296875,
-      "loss": 0.5703,
-      "rewards/accuracies": 0.596875011920929,
-      "rewards/chosen": -0.1341991424560547,
-      "rewards/margins": 1.1414505243301392,
-      "rewards/rejected": -1.27621328830719,
+      "logps/chosen": -259.2515563964844,
+      "logps/rejected": -317.92498779296875,
+      "loss": 0.6109,
+      "rewards/accuracies": 0.640625,
+      "rewards/chosen": -0.09950466454029083,
+      "rewards/margins": 0.8476117849349976,
+      "rewards/rejected": -0.9464820623397827,
       "step": 60
     },
     {
-      "epoch": 0.6553245116572148,
-      "grad_norm": 62.601972259800355,
-      "learning_rate": 1.4148936170212768e-07,
+      "epoch": 0.7460545193687231,
+      "grad_norm": 46.246379021501845,
+      "learning_rate": 1.3212121212121213e-07,
       "logits/chosen": NaN,
       "logits/rejected": NaN,
-      "logps/chosen": -358.22265625,
-      "logps/rejected": -460.4937438964844,
-      "loss": 0.5561,
-      "rewards/accuracies": 0.612500011920929,
-      "rewards/chosen": -0.09829378128051758,
-      "rewards/margins": 1.154931664466858,
-      "rewards/rejected": -1.25310218334198,
+      "logps/chosen": -258.0960998535156,
+      "logps/rejected": -290.5296936035156,
+      "loss": 0.617,
+      "rewards/accuracies": 0.640625,
+      "rewards/chosen": -0.1353795975446701,
+      "rewards/margins": 0.7194949984550476,
+      "rewards/rejected": -0.8551372289657593,
       "step": 65
     },
     {
-      "epoch": 0.7057340894770007,
-      "grad_norm": 42.81446910830512,
-      "learning_rate": 1.3617021276595742e-07,
+      "epoch": 0.8034433285509326,
+      "grad_norm": 51.298985955922575,
+      "learning_rate": 1.2606060606060603e-07,
       "logits/chosen": NaN,
       "logits/rejected": NaN,
-      "logps/chosen": -350.8062438964844,
-      "logps/rejected": -479.98126220703125,
-      "loss": 0.5464,
-      "rewards/accuracies": 0.668749988079071,
-      "rewards/chosen": -0.08439864963293076,
-      "rewards/margins": 1.0637038946151733,
-      "rewards/rejected": -1.146966576576233,
+      "logps/chosen": -290.3218688964844,
+      "logps/rejected": -378.1734313964844,
+      "loss": 0.5908,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -0.12958745658397675,
+      "rewards/margins": 0.8194991946220398,
+      "rewards/rejected": -0.9491798281669617,
       "step": 70
     },
     {
-      "epoch": 0.7561436672967864,
-      "grad_norm": 59.85465445918704,
-      "learning_rate": 1.3085106382978723e-07,
+      "epoch": 0.860832137733142,
+      "grad_norm": 51.37452979537066,
+      "learning_rate": 1.2e-07,
       "logits/chosen": NaN,
       "logits/rejected": NaN,
-      "logps/chosen": -339.3656311035156,
-      "logps/rejected": -428.65936279296875,
-      "loss": 0.5872,
-      "rewards/accuracies": 0.643750011920929,
-      "rewards/chosen": -0.11314620822668076,
-      "rewards/margins": 0.7480255365371704,
-      "rewards/rejected": -0.8610885739326477,
+      "logps/chosen": -272.2359313964844,
+      "logps/rejected": -461.9546813964844,
+      "loss": 0.6139,
+      "rewards/accuracies": 0.596875011920929,
+      "rewards/chosen": -0.08674906194210052,
+      "rewards/margins": 0.7904602289199829,
+      "rewards/rejected": -0.8770895004272461,
       "step": 75
     },
     {
-      "epoch": 0.8065532451165721,
-      "grad_norm": 77.89913196854276,
-      "learning_rate": 1.25531914893617e-07,
+      "epoch": 0.9182209469153515,
+      "grad_norm": 49.12644933327405,
+      "learning_rate": 1.1393939393939393e-07,
       "logits/chosen": NaN,
-      "logits/rejected": -0.232859805226326,
-      "logps/chosen": -367.71563720703125,
-      "logps/rejected": -496.9125061035156,
-      "loss": 0.5386,
-      "rewards/accuracies": 0.6968749761581421,
-      "rewards/chosen": -0.09202079474925995,
-      "rewards/margins": 1.1276824474334717,
-      "rewards/rejected": -1.2203514575958252,
+      "logits/rejected": NaN,
+      "logps/chosen": -271.92657470703125,
+      "logps/rejected": -385.4671936035156,
+      "loss": 0.5887,
+      "rewards/accuracies": 0.6343749761581421,
+      "rewards/chosen": -0.12120027840137482,
+      "rewards/margins": 0.9173402786254883,
+      "rewards/rejected": -1.0387518405914307,
       "step": 80
     },
     {
-      "epoch": 0.856962822936358,
-      "grad_norm": 51.576853346068596,
-      "learning_rate": 1.202127659574468e-07,
+      "epoch": 0.975609756097561,
+      "grad_norm": 78.67875740741145,
+      "learning_rate": 1.0787878787878789e-07,
       "logits/chosen": NaN,
       "logits/rejected": NaN,
-      "logps/chosen": -352.01873779296875,
-      "logps/rejected": -441.85626220703125,
-      "loss": 0.5489,
-      "rewards/accuracies": 0.699999988079071,
-      "rewards/chosen": -0.09004707634449005,
-      "rewards/margins": 0.9977798461914062,
-      "rewards/rejected": -1.0878921747207642,
+      "logps/chosen": -268.3734436035156,
+      "logps/rejected": -356.06561279296875,
+      "loss": 0.6094,
+      "rewards/accuracies": 0.621874988079071,
+      "rewards/chosen": -0.08560104668140411,
+      "rewards/margins": 0.7130492925643921,
+      "rewards/rejected": -0.7986106872558594,
       "step": 85
     },
     {
-      "epoch": 0.9073724007561437,
-      "grad_norm": 51.91948881188928,
-      "learning_rate": 1.148936170212766e-07,
+      "epoch": 1.0329985652797704,
+      "grad_norm": 37.99397160982797,
+      "learning_rate": 1.018181818181818e-07,
       "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": -391.98748779296875,
-      "logps/rejected": -501.89373779296875,
-      "loss": 0.5429,
-      "rewards/accuracies": 0.668749988079071,
-      "rewards/chosen": -0.13425922393798828,
-      "rewards/margins": 1.2032638788223267,
-      "rewards/rejected": -1.339324951171875,
+      "logits/rejected": -0.2726287841796875,
+      "logps/chosen": -278.57501220703125,
+      "logps/rejected": -375.9390563964844,
+      "loss": 0.5887,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.10773544013500214,
+      "rewards/margins": 0.7839363217353821,
+      "rewards/rejected": -0.892169177532196,
       "step": 90
     },
     {
-      "epoch": 0.9577819785759294,
-      "grad_norm": 54.81194235391722,
-      "learning_rate": 1.0957446808510638e-07,
+      "epoch": 1.0903873744619799,
+      "grad_norm": 38.13934723090496,
+      "learning_rate": 9.575757575757574e-08,
       "logits/chosen": NaN,
       "logits/rejected": NaN,
-      "logps/chosen": -380.8531188964844,
-      "logps/rejected": -485.4375,
-      "loss": 0.5506,
-      "rewards/accuracies": 0.6875,
-      "rewards/chosen": -0.123291015625,
-      "rewards/margins": 1.137838363647461,
-      "rewards/rejected": -1.2604999542236328,
+      "logps/chosen": -287.20001220703125,
+      "logps/rejected": -351.8617248535156,
+      "loss": 0.5602,
+      "rewards/accuracies": 0.684374988079071,
+      "rewards/chosen": -0.09619579464197159,
+      "rewards/margins": 0.9326726794242859,
+      "rewards/rejected": -1.0286362171173096,
       "step": 95
     },
     {
-      "epoch": 1.0081915563957151,
-      "grad_norm": 51.52692462343263,
-      "learning_rate": 1.0425531914893617e-07,
+      "epoch": 1.1477761836441893,
+      "grad_norm": 59.66676077313003,
+      "learning_rate": 8.96969696969697e-08,
       "logits/chosen": NaN,
       "logits/rejected": NaN,
-      "logps/chosen": -357.92498779296875,
-      "logps/rejected": -420.00311279296875,
-      "loss": 0.5433,
-      "rewards/accuracies": 0.6875,
-      "rewards/chosen": -0.118899405002594,
-      "rewards/margins": 1.2265655994415283,
-      "rewards/rejected": -1.3462097644805908,
+      "logps/chosen": -253.2980499267578,
+      "logps/rejected": -323.6937561035156,
+      "loss": 0.5715,
+      "rewards/accuracies": 0.721875011920929,
+      "rewards/chosen": -0.09653882682323456,
+      "rewards/margins": 0.7721735239028931,
+      "rewards/rejected": -0.8687639236450195,
       "step": 100
     },
     {
-      "epoch": 1.0586011342155008,
-      "grad_norm": 48.23673648203366,
-      "learning_rate": 9.893617021276596e-08,
+      "epoch": 1.2051649928263988,
+      "grad_norm": 40.181942906152656,
+      "learning_rate": 8.363636363636363e-08,
       "logits/chosen": NaN,
       "logits/rejected": NaN,
-      "logps/chosen": -420.59375,
-      "logps/rejected": -551.4156494140625,
-      "loss": 0.4711,
-      "rewards/accuracies": 0.715624988079071,
-      "rewards/chosen": -0.05655860900878906,
-      "rewards/margins": 1.65283203125,
-      "rewards/rejected": -1.7101104259490967,
+      "logps/chosen": -269.91796875,
+      "logps/rejected": -342.6031188964844,
+      "loss": 0.5733,
+      "rewards/accuracies": 0.6781250238418579,
+      "rewards/chosen": -0.1024196594953537,
+      "rewards/margins": 0.6281814575195312,
+      "rewards/rejected": -0.7303474545478821,
       "step": 105
     },
     {
-      "epoch": 1.1090107120352868,
-      "grad_norm": 129.8626362436788,
-      "learning_rate": 9.361702127659574e-08,
+      "epoch": 1.2625538020086085,
+      "grad_norm": 43.130194806696174,
+      "learning_rate": 7.757575757575757e-08,
       "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": -375.9306640625,
-      "logps/rejected": -410.8500061035156,
-      "loss": 0.5499,
-      "rewards/accuracies": 0.706250011920929,
-      "rewards/chosen": -0.10003051906824112,
-      "rewards/margins": 0.8063720464706421,
-      "rewards/rejected": -0.906982421875,
+      "logits/rejected": -0.28910523653030396,
+      "logps/chosen": -256.4398498535156,
+      "logps/rejected": -315.8851623535156,
+      "loss": 0.5609,
+      "rewards/accuracies": 0.7406250238418579,
+      "rewards/chosen": -0.05760955810546875,
+      "rewards/margins": 0.8904060125350952,
+      "rewards/rejected": -0.948272705078125,
       "step": 110
     },
     {
-      "epoch": 1.1594202898550725,
-      "grad_norm": 37.96460632662538,
-      "learning_rate": 8.829787234042553e-08,
+      "epoch": 1.3199426111908177,
+      "grad_norm": 40.187136674467965,
+      "learning_rate": 7.151515151515152e-08,
       "logits/chosen": NaN,
       "logits/rejected": NaN,
-      "logps/chosen": -395.4546813964844,
-      "logps/rejected": -534.5437622070312,
-      "loss": 0.4536,
-      "rewards/accuracies": 0.7875000238418579,
-      "rewards/chosen": -0.0355035774409771,
-      "rewards/margins": 1.6931426525115967,
-      "rewards/rejected": -1.7283508777618408,
+      "logps/chosen": -286.8125,
+      "logps/rejected": -305.38751220703125,
+      "loss": 0.5859,
+      "rewards/accuracies": 0.684374988079071,
+      "rewards/chosen": -0.05495605617761612,
+      "rewards/margins": 0.7184921503067017,
+      "rewards/rejected": -0.7738761901855469,
       "step": 115
     },
     {
-      "epoch": 1.2098298676748582,
-      "grad_norm": 41.75349141526311,
-      "learning_rate": 8.297872340425531e-08,
+      "epoch": 1.3773314203730274,
+      "grad_norm": 62.26850922502401,
+      "learning_rate": 6.545454545454545e-08,
       "logits/chosen": NaN,
-      "logits/rejected": -0.24345549941062927,
-      "logps/chosen": -337.83905029296875,
-      "logps/rejected": -414.4765625,
-      "loss": 0.4953,
-      "rewards/accuracies": 0.796875,
-      "rewards/chosen": -0.09084253013134003,
-      "rewards/margins": 1.2055069208145142,
-      "rewards/rejected": -1.2963898181915283,
+      "logits/rejected": NaN,
+      "logps/chosen": -281.8828125,
+      "logps/rejected": -348.609375,
+      "loss": 0.5641,
+      "rewards/accuracies": 0.715624988079071,
+      "rewards/chosen": -0.0843501091003418,
+      "rewards/margins": 0.8600600957870483,
+      "rewards/rejected": -0.9451843500137329,
       "step": 120
     },
     {
-      "epoch": 1.260239445494644,
-      "grad_norm": 47.57568984542951,
-      "learning_rate": 7.76595744680851e-08,
+      "epoch": 1.4347202295552366,
+      "grad_norm": 40.00896779584937,
+      "learning_rate": 5.93939393939394e-08,
       "logits/chosen": NaN,
       "logits/rejected": NaN,
-      "logps/chosen": -358.9046936035156,
-      "logps/rejected": -469.49688720703125,
-      "loss": 0.501,
-      "rewards/accuracies": 0.75,
-      "rewards/chosen": -0.09036216884851456,
-      "rewards/margins": 1.231683373451233,
-      "rewards/rejected": -1.3224579095840454,
+      "logps/chosen": -289.64959716796875,
+      "logps/rejected": -378.01873779296875,
+      "loss": 0.5644,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -0.06678199768066406,
+      "rewards/margins": 0.8301635980606079,
+      "rewards/rejected": -0.8972938656806946,
       "step": 125
     },
     {
-      "epoch": 1.3106490233144297,
-      "grad_norm": 53.944830915941765,
-      "learning_rate": 7.23404255319149e-08,
+      "epoch": 1.4921090387374463,
+      "grad_norm": 33.42224905779865,
+      "learning_rate": 5.333333333333333e-08,
       "logits/chosen": NaN,
       "logits/rejected": NaN,
-      "logps/chosen": -356.4375,
-      "logps/rejected": -489.125,
-      "loss": 0.4999,
+      "logps/chosen": -228.01406860351562,
+      "logps/rejected": -385.5078125,
+      "loss": 0.5231,
       "rewards/accuracies": 0.7593749761581421,
-      "rewards/chosen": -0.0903778076171875,
-      "rewards/margins": 1.2274360656738281,
-      "rewards/rejected": -1.3174560070037842,
+      "rewards/chosen": -0.06652259826660156,
+      "rewards/margins": 1.106951117515564,
+      "rewards/rejected": -1.173893690109253,
       "step": 130
     },
     {
-      "epoch": 1.3610586011342156,
-      "grad_norm": 40.45839840494219,
-      "learning_rate": 6.702127659574469e-08,
+      "epoch": 1.5494978479196555,
+      "grad_norm": 35.377034170414966,
+      "learning_rate": 4.727272727272727e-08,
       "logits/chosen": NaN,
       "logits/rejected": NaN,
-      "logps/chosen": -368.95623779296875,
-      "logps/rejected": -460.58282470703125,
-      "loss": 0.4856,
-      "rewards/accuracies": 0.746874988079071,
-      "rewards/chosen": -0.033612824976444244,
-      "rewards/margins": 1.4452941417694092,
-      "rewards/rejected": -1.479437232017517,
+      "logps/chosen": -276.4312438964844,
+      "logps/rejected": -374.94219970703125,
+      "loss": 0.5493,
+      "rewards/accuracies": 0.753125011920929,
+      "rewards/chosen": -0.07003593444824219,
+      "rewards/margins": 0.8382889032363892,
+      "rewards/rejected": -0.9086562991142273,
       "step": 135
     },
     {
-      "epoch": 1.4114681789540013,
-      "grad_norm": 43.63902778407327,
-      "learning_rate": 6.170212765957446e-08,
+      "epoch": 1.6068866571018652,
+      "grad_norm": 42.98104883723141,
+      "learning_rate": 4.121212121212121e-08,
       "logits/chosen": NaN,
       "logits/rejected": NaN,
-      "logps/chosen": -393.62030029296875,
-      "logps/rejected": -473.53436279296875,
-      "loss": 0.4987,
-      "rewards/accuracies": 0.734375,
-      "rewards/chosen": -0.03872375562787056,
-      "rewards/margins": 1.239990234375,
-      "rewards/rejected": -1.2792266607284546,
+      "logps/chosen": -245.8046875,
+      "logps/rejected": -332.34844970703125,
+      "loss": 0.5553,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -0.09261999279260635,
+      "rewards/margins": 0.8664749264717102,
+      "rewards/rejected": -0.9593642950057983,
       "step": 140
     },
     {
-      "epoch": 1.461877756773787,
-      "grad_norm": 40.93200179183777,
-      "learning_rate": 5.638297872340425e-08,
+      "epoch": 1.6642754662840746,
+      "grad_norm": 41.194723091835506,
+      "learning_rate": 3.5151515151515146e-08,
       "logits/chosen": NaN,
-      "logits/rejected": -0.28594666719436646,
-      "logps/chosen": -370.62188720703125,
-      "logps/rejected": -508.2562561035156,
-      "loss": 0.4891,
-      "rewards/accuracies": 0.7250000238418579,
-      "rewards/chosen": -0.047638703137636185,
-      "rewards/margins": 1.3118622303009033,
-      "rewards/rejected": -1.359655737876892,
+      "logits/rejected": NaN,
+      "logps/chosen": -271.9007873535156,
+      "logps/rejected": -372.2890625,
+      "loss": 0.576,
+      "rewards/accuracies": 0.703125,
+      "rewards/chosen": -0.10253047943115234,
+      "rewards/margins": 0.6510879397392273,
+      "rewards/rejected": -0.7538429498672485,
       "step": 145
     },
     {
-      "epoch": 1.5122873345935728,
-      "grad_norm": 58.65309693932161,
-      "learning_rate": 5.106382978723404e-08,
+      "epoch": 1.721664275466284,
+      "grad_norm": 44.50624051602098,
+      "learning_rate": 2.9090909090909088e-08,
       "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": -347.8609313964844,
-      "logps/rejected": -473.4750061035156,
-      "loss": 0.4814,
-      "rewards/accuracies": 0.762499988079071,
-      "rewards/chosen": -0.05366211012005806,
-      "rewards/margins": 1.2643524408340454,
-      "rewards/rejected": -1.3187682628631592,
+      "logits/rejected": -0.3272903561592102,
+      "logps/chosen": -241.9031219482422,
+      "logps/rejected": -336.61407470703125,
+      "loss": 0.557,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.09629325568675995,
+      "rewards/margins": 0.7580966949462891,
+      "rewards/rejected": -0.8545807003974915,
       "step": 150
     },
     {
-      "epoch": 1.5626969124133585,
-      "grad_norm": 40.5990127283572,
-      "learning_rate": 4.5744680851063826e-08,
+      "epoch": 1.7790530846484935,
+      "grad_norm": 35.414071145451,
+      "learning_rate": 2.303030303030303e-08,
       "logits/chosen": NaN,
       "logits/rejected": NaN,
-      "logps/chosen": -377.1734313964844,
-      "logps/rejected": -453.3296813964844,
-      "loss": 0.4998,
-      "rewards/accuracies": 0.753125011920929,
-      "rewards/chosen": -0.07384242862462997,
-      "rewards/margins": 1.2074543237686157,
-      "rewards/rejected": -1.2816162109375,
+      "logps/chosen": -249.95938110351562,
+      "logps/rejected": -301.9765625,
+      "loss": 0.5675,
+      "rewards/accuracies": 0.778124988079071,
+      "rewards/chosen": -0.08613376319408417,
+      "rewards/margins": 0.8244568109512329,
+      "rewards/rejected": -0.9111496210098267,
       "step": 155
     },
     {
-      "epoch": 1.6131064902331445,
-      "grad_norm": 32.42610640703847,
-      "learning_rate": 4.0425531914893614e-08,
+      "epoch": 1.836441893830703,
+      "grad_norm": 38.330459518863954,
+      "learning_rate": 1.696969696969697e-08,
       "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": -330.2578125,
-      "logps/rejected": -398.8812561035156,
-      "loss": 0.471,
-      "rewards/accuracies": 0.7906249761581421,
-      "rewards/chosen": -0.01856536790728569,
-      "rewards/margins": 1.3712249994277954,
-      "rewards/rejected": -1.3899352550506592,
+      "logits/rejected": -0.393698126077652,
+      "logps/chosen": -262.72186279296875,
+      "logps/rejected": -394.54217529296875,
+      "loss": 0.5666,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -0.09622383117675781,
+      "rewards/margins": 0.8662067651748657,
+      "rewards/rejected": -0.9627344012260437,
       "step": 160
     },
     {
-      "epoch": 1.66351606805293,
-      "grad_norm": 41.01530072372472,
-      "learning_rate": 3.51063829787234e-08,
+      "epoch": 1.8938307030129125,
+      "grad_norm": 41.39148819542555,
+      "learning_rate": 1.0909090909090908e-08,
       "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": -340.2749938964844,
-      "logps/rejected": -438.40313720703125,
-      "loss": 0.4842,
-      "rewards/accuracies": 0.737500011920929,
-      "rewards/chosen": -0.0492522232234478,
-      "rewards/margins": 1.4962584972381592,
-      "rewards/rejected": -1.5454528331756592,
+      "logits/rejected": -0.3541931211948395,
+      "logps/chosen": -286.5625,
+      "logps/rejected": -391.0406188964844,
+      "loss": 0.5354,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.07141885906457901,
+      "rewards/margins": 1.0486961603164673,
+      "rewards/rejected": -1.120294213294983,
       "step": 165
     },
     {
-      "epoch": 1.713925645872716,
-      "grad_norm": 40.64613349590343,
-      "learning_rate": 2.9787234042553187e-08,
+      "epoch": 1.951219512195122,
+      "grad_norm": 39.2547263324904,
+      "learning_rate": 4.848484848484848e-09,
       "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": -348.5640563964844,
-      "logps/rejected": -449.84844970703125,
-      "loss": 0.4915,
-      "rewards/accuracies": 0.731249988079071,
-      "rewards/chosen": -0.07779388129711151,
-      "rewards/margins": 1.278845191001892,
-      "rewards/rejected": -1.356591820716858,
+      "logits/rejected": -0.39358216524124146,
+      "logps/chosen": -267.47265625,
+      "logps/rejected": -372.5625,
+      "loss": 0.5478,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -0.05818195268511772,
+      "rewards/margins": 1.0078842639923096,
+      "rewards/rejected": -1.065637230873108,
       "step": 170
     },
     {
-      "epoch": 1.7643352236925016,
-      "grad_norm": 41.86507518705149,
-      "learning_rate": 2.4468085106382976e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": -350.2046813964844,
-      "logps/rejected": -433.5687561035156,
-      "loss": 0.4755,
-      "rewards/accuracies": 0.8031250238418579,
-      "rewards/chosen": -0.027071380987763405,
-      "rewards/margins": 1.4462082386016846,
-      "rewards/rejected": -1.4738037586212158,
-      "step": 175
-    },
-    {
-      "epoch": 1.8147448015122873,
-      "grad_norm": 46.67757830253006,
-      "learning_rate": 1.9148936170212764e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": -398.32501220703125,
-      "logps/rejected": -456.9437561035156,
-      "loss": 0.4807,
-      "rewards/accuracies": 0.746874988079071,
-      "rewards/chosen": -0.05414886400103569,
-      "rewards/margins": 1.2570632696151733,
-      "rewards/rejected": -1.310980200767517,
-      "step": 180
-    },
-    {
-      "epoch": 1.865154379332073,
-      "grad_norm": 43.636875496682755,
-      "learning_rate": 1.3829787234042552e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": -0.17527160048484802,
-      "logps/chosen": -373.87188720703125,
-      "logps/rejected": -535.796875,
-      "loss": 0.4376,
-      "rewards/accuracies": 0.796875,
-      "rewards/chosen": -0.026004791259765625,
-      "rewards/margins": 1.6229279041290283,
-      "rewards/rejected": -1.649743676185608,
-      "step": 185
-    },
-    {
-      "epoch": 1.9155639571518588,
-      "grad_norm": 41.753332033000945,
-      "learning_rate": 8.510638297872339e-09,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": -360.0718688964844,
-      "logps/rejected": -468.2124938964844,
-      "loss": 0.4872,
-      "rewards/accuracies": 0.731249988079071,
-      "rewards/chosen": -0.11498375236988068,
-      "rewards/margins": 1.2304840087890625,
-      "rewards/rejected": -1.345800757408142,
-      "step": 190
-    },
-    {
-      "epoch": 1.9659735349716447,
-      "grad_norm": 43.377545693326894,
-      "learning_rate": 3.1914893617021273e-09,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": -365.515625,
-      "logps/rejected": -478.56561279296875,
-      "loss": 0.4805,
-      "rewards/accuracies": 0.7718750238418579,
-      "rewards/chosen": -0.04032173007726669,
-      "rewards/margins": 1.3543853759765625,
-      "rewards/rejected": -1.39520263671875,
-      "step": 195
-    },
-    {
-      "epoch": 1.996219281663516,
-      "step": 198,
+      "epoch": 1.9971305595408895,
+      "step": 174,
       "total_flos": 0.0,
-      "train_loss": 0.5393935610549618,
-      "train_runtime": 2398.307,
-      "train_samples_per_second": 5.293,
-      "train_steps_per_second": 0.083
+      "train_loss": 0.5929731760901966,
+      "train_runtime": 2030.2211,
+      "train_samples_per_second": 5.493,
+      "train_steps_per_second": 0.086
     }
   ],
   "logging_steps": 5,
-  "max_steps": 198,
+  "max_steps": 174,
   "num_input_tokens_seen": 0,
   "num_train_epochs": 2,
   "save_steps": 50,

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:04392d39f00478df0417f7fd9ba7d15085f3dba381f41d035a42742e674b52ff
+oid sha256:2c63478df6f2cc0a8edc66dba8ca06784108c3576df8e676705056d7247719f8
 size 7672