{ "best_global_step": null, "best_metric": null, "best_model_checkpoint": null, "epoch": 0.009184423218221896, "eval_steps": 10, "global_step": 100, "is_hyper_param_search": false, "is_local_process_zero": true, "is_world_process_zero": true, "log_history": [ { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1911.0, "completions/max_terminated_length": 1911.0, "completions/mean_length": 737.8, "completions/mean_terminated_length": 737.8, "completions/min_length": 9.0, "completions/min_terminated_length": 9.0, "epoch": 9.184423218221895e-05, "grad_norm": 2.7607495142433, "kl": 0.00098419189453125, "learning_rate": 0.0, "loss": -0.6015, "num_tokens": 43442.0, "reward": 0.15676024556159973, "reward_std": 0.0992189422249794, "rewards/format_reward_func": 0.6000000238418579, "rewards/lcs_reward_func": 0.03676025941967964, "step": 1 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "epoch": 0.0001836884643644379, "grad_norm": 2.7607084486431908, "kl": 0.00098419189453125, "learning_rate": 1e-07, "loss": -0.6015, "step": 2 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1826.0, "completions/max_terminated_length": 1826.0, "completions/mean_length": 739.525, "completions/mean_terminated_length": 739.525, "completions/min_length": 9.0, "completions/min_terminated_length": 9.0, "epoch": 0.0002755326965466569, "grad_norm": 2.5034917199753006, "kl": 0.0010004043579101562, "learning_rate": 2e-07, "loss": -0.543, "num_tokens": 85793.0, "reward": 0.15654990077018738, "reward_std": 0.09047843515872955, "rewards/format_reward_func": 0.6500000357627869, "rewards/lcs_reward_func": 0.02654988504946232, "step": 3 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "epoch": 0.0003673769287288758, "grad_norm": 2.5049494762583047, "kl": 0.001041412353515625, "learning_rate": 3e-07, "loss": -0.543, "step": 4 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1984.0, "completions/max_terminated_length": 1984.0, "completions/mean_length": 722.925, "completions/mean_terminated_length": 722.925, "completions/min_length": 9.0, "completions/min_terminated_length": 9.0, "epoch": 0.0004592211609110948, "grad_norm": 3.0465073005843717, "kl": 0.0007085800170898438, "learning_rate": 4e-07, "loss": -0.6571, "num_tokens": 127850.0, "reward": 0.15640318393707275, "reward_std": 0.08817142993211746, "rewards/format_reward_func": 0.6700000166893005, "rewards/lcs_reward_func": 0.022403182461857796, "step": 5 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "epoch": 0.0005510653930933138, "grad_norm": 3.0448516969487267, "kl": 0.000705718994140625, "learning_rate": 5e-07, "loss": -0.6571, "step": 6 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 2029.0, "completions/max_terminated_length": 2029.0, "completions/mean_length": 894.325, "completions/mean_terminated_length": 894.325, "completions/min_length": 9.0, "completions/min_terminated_length": 9.0, "epoch": 0.0006429096252755327, "grad_norm": 1.8772862682834104, "kl": 0.0007915496826171875, "learning_rate": 6e-07, "loss": -0.3071, "num_tokens": 176853.0, "reward": 0.201990008354187, "reward_std": 0.11269382387399673, "rewards/format_reward_func": 0.6550000309944153, "rewards/lcs_reward_func": 0.07099001854658127, "step": 7 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "epoch": 0.0007347538574577516, "grad_norm": 1.8735448714531584, "kl": 0.0007781982421875, "learning_rate": 7e-07, "loss": -0.3071, "step": 8 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1736.0, "completions/max_terminated_length": 1736.0, "completions/mean_length": 886.15, "completions/mean_terminated_length": 886.15, "completions/min_length": 9.0, "completions/min_terminated_length": 9.0, "epoch": 0.0008265980896399706, "grad_norm": 1.8594898973134009, "kl": 0.000946044921875, "learning_rate": 8e-07, "loss": -0.4133, "num_tokens": 226519.0, "reward": 0.17254838347434998, "reward_std": 0.08416791260242462, "rewards/format_reward_func": 0.6950000524520874, "rewards/lcs_reward_func": 0.03354838490486145, "step": 9 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "epoch": 0.0009184423218221896, "grad_norm": 1.8465168294754166, "kl": 0.0010614395141601562, "learning_rate": 9e-07, "loss": -0.4133, "step": 10 }, { "epoch": 0.0009184423218221896, "eval_completions/max_length": 2004, "eval_completions/mean_length": 771.96875, "eval_completions/min_length": 33, "eval_reward": 0.21109572052955627, "eval_reward_std": 0.11327587813138962, "step": 10 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 2027.0, "completions/max_terminated_length": 2027.0, "completions/mean_length": 780.475, "completions/mean_terminated_length": 780.475, "completions/min_length": 9.0, "completions/min_terminated_length": 9.0, "epoch": 0.0010102865540044085, "grad_norm": 2.231378663403048, "kl": 0.0013132095336914062, "learning_rate": 1e-06, "loss": -0.4691, "num_tokens": 270268.0, "reward": 0.18357333540916443, "reward_std": 0.09895642846822739, "rewards/format_reward_func": 0.6800000071525574, "rewards/lcs_reward_func": 0.04757332801818848, "step": 11 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "epoch": 0.0011021307861866275, "grad_norm": 2.203346987626314, "kl": 0.00273895263671875, "learning_rate": 1e-06, "loss": -0.4691, "step": 12 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 2046.0, "completions/max_terminated_length": 2046.0, "completions/mean_length": 886.75, "completions/mean_terminated_length": 886.75, "completions/min_length": 9.0, "completions/min_terminated_length": 9.0, "epoch": 0.0011939750183688465, "grad_norm": 1.722693626980397, "kl": 0.0030231475830078125, "learning_rate": 1e-06, "loss": -0.4224, "num_tokens": 318068.0, "reward": 0.20494212210178375, "reward_std": 0.09967484325170517, "rewards/format_reward_func": 0.7099999785423279, "rewards/lcs_reward_func": 0.06294212490320206, "step": 13 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "epoch": 0.0012858192505510655, "grad_norm": 1.71286565180573, "kl": 0.004039764404296875, "learning_rate": 1e-06, "loss": -0.4224, "step": 14 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1850.0, "completions/max_terminated_length": 1850.0, "completions/mean_length": 810.525, "completions/mean_terminated_length": 810.525, "completions/min_length": 9.0, "completions/min_terminated_length": 9.0, "epoch": 0.0013776634827332844, "grad_norm": 1.5003857131255895, "kl": 0.008074760437011719, "learning_rate": 1e-06, "loss": -0.1635, "num_tokens": 363389.0, "reward": 0.2573193907737732, "reward_std": 0.10465292632579803, "rewards/format_reward_func": 0.8100000619888306, "rewards/lcs_reward_func": 0.09531939774751663, "step": 15 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "epoch": 0.0014695077149155032, "grad_norm": 1.4613380324648746, "kl": 0.02646923065185547, "learning_rate": 1e-06, "loss": -0.1635, "step": 16 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1739.0, "completions/max_terminated_length": 1739.0, "completions/mean_length": 812.625, "completions/mean_terminated_length": 812.625, "completions/min_length": 9.0, "completions/min_terminated_length": 9.0, "epoch": 0.0015613519470977222, "grad_norm": 1.8041011323824336, "kl": 0.05150604248046875, "learning_rate": 1e-06, "loss": -0.1445, "num_tokens": 407234.0, "reward": 0.2477889060974121, "reward_std": 0.09424225986003876, "rewards/format_reward_func": 0.76500004529953, "rewards/lcs_reward_func": 0.09478890895843506, "step": 17 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "epoch": 0.0016531961792799412, "grad_norm": 1.8014091360654865, "kl": 0.09454345703125, "learning_rate": 1e-06, "loss": -0.1444, "step": 18 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1855.0, "completions/max_terminated_length": 1855.0, "completions/mean_length": 662.15, "completions/mean_terminated_length": 662.15, "completions/min_length": 9.0, "completions/min_terminated_length": 9.0, "epoch": 0.0017450404114621601, "grad_norm": 2.720704647358892, "kl": 0.529296875, "learning_rate": 1e-06, "loss": -0.6313, "num_tokens": 446840.0, "reward": 0.1577414870262146, "reward_std": 0.07978807389736176, "rewards/format_reward_func": 0.6700000166893005, "rewards/lcs_reward_func": 0.02374148927628994, "step": 19 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "epoch": 0.0018368846436443791, "grad_norm": 2.7053569037359777, "kl": 0.82177734375, "learning_rate": 1e-06, "loss": -0.6311, "step": 20 }, { "epoch": 0.0018368846436443791, "eval_completions/max_length": 2065, "eval_completions/mean_length": 889.28125, "eval_completions/min_length": 33, "eval_reward": 0.2044806331396103, "eval_reward_std": 0.09299344569444656, "step": 20 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 2021.0, "completions/max_terminated_length": 2021.0, "completions/mean_length": 655.925, "completions/mean_terminated_length": 655.925, "completions/min_length": 9.0, "completions/min_terminated_length": 9.0, "epoch": 0.001928728875826598, "grad_norm": 3.0706531919007047, "kl": 1.38525390625, "learning_rate": 1e-06, "loss": -0.856, "num_tokens": 485717.0, "reward": 0.1469292938709259, "reward_std": 0.10327912867069244, "rewards/format_reward_func": 0.5800000429153442, "rewards/lcs_reward_func": 0.03092929720878601, "step": 21 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "epoch": 0.002020573108008817, "grad_norm": 3.0607964501913987, "kl": 1.6064453125, "learning_rate": 1e-06, "loss": -0.8558, "step": 22 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1959.0, "completions/max_terminated_length": 1959.0, "completions/mean_length": 863.425, "completions/mean_terminated_length": 863.425, "completions/min_length": 5.0, "completions/min_terminated_length": 5.0, "epoch": 0.002112417340191036, "grad_norm": 1.4283474171915316, "kl": 0.6814689636230469, "learning_rate": 1e-06, "loss": -0.4158, "num_tokens": 532504.0, "reward": 0.1960056722164154, "reward_std": 0.07993875443935394, "rewards/format_reward_func": 0.7649999856948853, "rewards/lcs_reward_func": 0.04300567880272865, "step": 23 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "epoch": 0.002204261572373255, "grad_norm": 1.4416929903577083, "kl": 1.0042381286621094, "learning_rate": 1e-06, "loss": -0.4154, "step": 24 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1924.0, "completions/max_terminated_length": 1924.0, "completions/mean_length": 890.625, "completions/mean_terminated_length": 890.625, "completions/min_length": 9.0, "completions/min_terminated_length": 9.0, "epoch": 0.002296105804555474, "grad_norm": 1.6827770351938804, "kl": 0.92138671875, "learning_rate": 1e-06, "loss": -0.2923, "num_tokens": 580859.0, "reward": 0.21194560825824738, "reward_std": 0.0956936851143837, "rewards/format_reward_func": 0.7800000309944153, "rewards/lcs_reward_func": 0.05594560503959656, "step": 25 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "epoch": 0.002387950036737693, "grad_norm": 1.7236721170439555, "kl": 1.6044921875, "learning_rate": 1e-06, "loss": -0.2916, "step": 26 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 2044.0, "completions/max_terminated_length": 2044.0, "completions/mean_length": 916.55, "completions/mean_terminated_length": 916.55, "completions/min_length": 9.0, "completions/min_terminated_length": 9.0, "epoch": 0.002479794268919912, "grad_norm": 4.19058051585696, "kl": 5.715314865112305, "learning_rate": 1e-06, "loss": -0.2778, "num_tokens": 629931.0, "reward": 0.20700404047966003, "reward_std": 0.09488054364919662, "rewards/format_reward_func": 0.7200000286102295, "rewards/lcs_reward_func": 0.0630040392279625, "step": 27 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "epoch": 0.002571638501102131, "grad_norm": 4.1982741479661625, "kl": 6.281728744506836, "learning_rate": 1e-06, "loss": -0.2772, "step": 28 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1817.0, "completions/max_terminated_length": 1817.0, "completions/mean_length": 837.875, "completions/mean_terminated_length": 837.875, "completions/min_length": 9.0, "completions/min_terminated_length": 9.0, "epoch": 0.00266348273328435, "grad_norm": 7.077824440439472, "kl": 12.953125, "learning_rate": 1e-06, "loss": -0.3563, "num_tokens": 677336.0, "reward": 0.21692918241024017, "reward_std": 0.10151036083698273, "rewards/format_reward_func": 0.8100000619888306, "rewards/lcs_reward_func": 0.05492917448282242, "step": 29 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "epoch": 0.002755326965466569, "grad_norm": 7.8992652256415425, "kl": 14.34765625, "learning_rate": 1e-06, "loss": -0.355, "step": 30 }, { "epoch": 0.002755326965466569, "eval_completions/max_length": 1941, "eval_completions/mean_length": 1021.4375, "eval_completions/min_length": 33, "eval_reward": 0.21428458392620087, "eval_reward_std": 0.07939482480287552, "step": 30 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 2037.0, "completions/max_terminated_length": 2037.0, "completions/mean_length": 1040.2, "completions/mean_terminated_length": 1040.2, "completions/min_length": 9.0, "completions/min_terminated_length": 9.0, "epoch": 0.002847171197648788, "grad_norm": 1.6919689671443403, "kl": 2.2836875915527344, "learning_rate": 1e-06, "loss": -0.2171, "num_tokens": 732334.0, "reward": 0.2230643928050995, "reward_std": 0.06873544305562973, "rewards/format_reward_func": 0.8800000548362732, "rewards/lcs_reward_func": 0.04706438630819321, "step": 31 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "epoch": 0.0029390154298310064, "grad_norm": 1.4919001050426044, "kl": 2.106945037841797, "learning_rate": 1e-06, "loss": -0.2173, "step": 32 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1880.0, "completions/max_terminated_length": 1880.0, "completions/mean_length": 908.5, "completions/mean_terminated_length": 908.5, "completions/min_length": 9.0, "completions/min_terminated_length": 9.0, "epoch": 0.0030308596620132254, "grad_norm": 1.5930416401398144, "kl": 3.513671875, "learning_rate": 1e-06, "loss": -0.3184, "num_tokens": 780784.0, "reward": 0.19843518733978271, "reward_std": 0.09002426266670227, "rewards/format_reward_func": 0.7999999523162842, "rewards/lcs_reward_func": 0.0384351909160614, "step": 33 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "epoch": 0.0031227038941954444, "grad_norm": 1.5242817608935546, "kl": 3.2734375, "learning_rate": 1e-06, "loss": -0.3186, "step": 34 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1991.0, "completions/max_terminated_length": 1991.0, "completions/mean_length": 865.9, "completions/mean_terminated_length": 865.9, "completions/min_length": 9.0, "completions/min_terminated_length": 9.0, "epoch": 0.0032145481263776633, "grad_norm": 1.6408098250505057, "kl": 4.494140625, "learning_rate": 1e-06, "loss": -0.4562, "num_tokens": 831850.0, "reward": 0.1844530701637268, "reward_std": 0.08404044806957245, "rewards/format_reward_func": 0.7550000548362732, "rewards/lcs_reward_func": 0.0334530845284462, "step": 35 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "epoch": 0.0033063923585598823, "grad_norm": 1.4204649760811243, "kl": 4.013671875, "learning_rate": 1e-06, "loss": -0.4566, "step": 36 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 2036.0, "completions/max_terminated_length": 2036.0, "completions/mean_length": 985.825, "completions/mean_terminated_length": 985.825, "completions/min_length": 9.0, "completions/min_terminated_length": 9.0, "epoch": 0.0033982365907421013, "grad_norm": 1.435337921695266, "kl": 3.529296875, "learning_rate": 1e-06, "loss": -0.4318, "num_tokens": 886073.0, "reward": 0.19459547102451324, "reward_std": 0.09948933124542236, "rewards/format_reward_func": 0.7700000405311584, "rewards/lcs_reward_func": 0.04059547185897827, "step": 37 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "epoch": 0.0034900808229243203, "grad_norm": 1.5637950407978138, "kl": 3.4580078125, "learning_rate": 1e-06, "loss": -0.4319, "step": 38 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1984.0, "completions/max_terminated_length": 1984.0, "completions/mean_length": 870.6, "completions/mean_terminated_length": 870.6, "completions/min_length": 9.0, "completions/min_terminated_length": 9.0, "epoch": 0.0035819250551065392, "grad_norm": 1.3550377236404076, "kl": 2.017578125, "learning_rate": 1e-06, "loss": -0.2469, "num_tokens": 933597.0, "reward": 0.21190586686134338, "reward_std": 0.07402609288692474, "rewards/format_reward_func": 0.8350000381469727, "rewards/lcs_reward_func": 0.04490587115287781, "step": 39 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "epoch": 0.0036737692872887582, "grad_norm": 1.3403329530602544, "kl": 2.021484375, "learning_rate": 1e-06, "loss": -0.2469, "step": 40 }, { "epoch": 0.0036737692872887582, "eval_completions/max_length": 2028, "eval_completions/mean_length": 869.28125, "eval_completions/min_length": 33, "eval_reward": 0.20918260514736176, "eval_reward_std": 0.08579342067241669, "step": 40 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 2026.0, "completions/max_terminated_length": 2026.0, "completions/mean_length": 965.575, "completions/mean_terminated_length": 965.575, "completions/min_length": 9.0, "completions/min_terminated_length": 9.0, "epoch": 0.003765613519470977, "grad_norm": 1.486948705724367, "kl": 3.025390625, "learning_rate": 1e-06, "loss": -0.3514, "num_tokens": 984750.0, "reward": 0.20043790340423584, "reward_std": 0.07684105634689331, "rewards/format_reward_func": 0.800000011920929, "rewards/lcs_reward_func": 0.04043790325522423, "step": 41 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "epoch": 0.003857457751653196, "grad_norm": 1.5564521188030682, "kl": 3.048828125, "learning_rate": 1e-06, "loss": -0.3514, "step": 42 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 2023.0, "completions/max_terminated_length": 2023.0, "completions/mean_length": 962.35, "completions/mean_terminated_length": 962.35, "completions/min_length": 9.0, "completions/min_terminated_length": 9.0, "epoch": 0.003949301983835415, "grad_norm": 1.0825099833801732, "kl": 1.3337478637695312, "learning_rate": 1e-06, "loss": -0.2821, "num_tokens": 1035704.0, "reward": 0.23006513714790344, "reward_std": 0.08291836082935333, "rewards/format_reward_func": 0.8200001120567322, "rewards/lcs_reward_func": 0.06606514751911163, "step": 43 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "epoch": 0.004041146216017634, "grad_norm": 1.0843188590306336, "kl": 1.2898292541503906, "learning_rate": 1e-06, "loss": -0.2821, "step": 44 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1924.0, "completions/max_terminated_length": 1924.0, "completions/mean_length": 900.25, "completions/mean_terminated_length": 900.25, "completions/min_length": 9.0, "completions/min_terminated_length": 9.0, "epoch": 0.004132990448199853, "grad_norm": 1.5379484313730982, "kl": 3.1171875, "learning_rate": 1e-06, "loss": -0.4024, "num_tokens": 1084314.0, "reward": 0.20005948841571808, "reward_std": 0.09878475964069366, "rewards/format_reward_func": 0.7900000810623169, "rewards/lcs_reward_func": 0.04205949231982231, "step": 45 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "epoch": 0.004224834680382072, "grad_norm": 1.5124606590890506, "kl": 3.0166015625, "learning_rate": 1e-06, "loss": -0.4024, "step": 46 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1850.0, "completions/max_terminated_length": 1850.0, "completions/mean_length": 938.775, "completions/mean_terminated_length": 938.775, "completions/min_length": 9.0, "completions/min_terminated_length": 9.0, "epoch": 0.004316678912564291, "grad_norm": 1.408416490591962, "kl": 1.4979820251464844, "learning_rate": 1e-06, "loss": -0.2602, "num_tokens": 1135535.0, "reward": 0.2001209557056427, "reward_std": 0.06422517448663712, "rewards/format_reward_func": 0.8700000047683716, "rewards/lcs_reward_func": 0.026120955124497414, "step": 47 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "epoch": 0.00440852314474651, "grad_norm": 1.3929253504743904, "kl": 1.4970436096191406, "learning_rate": 1e-06, "loss": -0.2602, "step": 48 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1917.0, "completions/max_terminated_length": 1917.0, "completions/mean_length": 900.025, "completions/mean_terminated_length": 900.025, "completions/min_length": 9.0, "completions/min_terminated_length": 9.0, "epoch": 0.004500367376928729, "grad_norm": 1.2289751162438687, "kl": 2.583984375, "learning_rate": 1e-06, "loss": -0.4411, "num_tokens": 1183336.0, "reward": 0.1928170770406723, "reward_std": 0.08593375235795975, "rewards/format_reward_func": 0.8100000619888306, "rewards/lcs_reward_func": 0.030817076563835144, "step": 49 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "epoch": 0.004592211609110948, "grad_norm": 1.2085154906618722, "kl": 2.6875, "learning_rate": 1e-06, "loss": -0.441, "step": 50 }, { "epoch": 0.004592211609110948, "eval_completions/max_length": 1942, "eval_completions/mean_length": 898.71875, "eval_completions/min_length": 33, "eval_reward": 0.20390485227108002, "eval_reward_std": 0.07529351860284805, "step": 50 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1970.0, "completions/max_terminated_length": 1970.0, "completions/mean_length": 828.45, "completions/mean_terminated_length": 828.45, "completions/min_length": 9.0, "completions/min_terminated_length": 9.0, "epoch": 0.0046840558412931665, "grad_norm": 1.690419927569193, "kl": 3.8125, "learning_rate": 1e-06, "loss": -0.6029, "num_tokens": 1228914.0, "reward": 0.17726577818393707, "reward_std": 0.10230447351932526, "rewards/format_reward_func": 0.7300000190734863, "rewards/lcs_reward_func": 0.03126578405499458, "step": 51 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "epoch": 0.004775900073475386, "grad_norm": 1.6807220195714059, "kl": 4.25, "learning_rate": 1e-06, "loss": -0.6025, "step": 52 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 2032.0, "completions/max_terminated_length": 2032.0, "completions/mean_length": 1138.725, "completions/mean_terminated_length": 1138.725, "completions/min_length": 9.0, "completions/min_terminated_length": 9.0, "epoch": 0.0048677443056576045, "grad_norm": 0.9847388426456154, "kl": 2.0333099365234375, "learning_rate": 1e-06, "loss": -0.3001, "num_tokens": 1290023.0, "reward": 0.21287302672863007, "reward_std": 0.07877419143915176, "rewards/format_reward_func": 0.8600000739097595, "rewards/lcs_reward_func": 0.04087302088737488, "step": 53 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "epoch": 0.004959588537839824, "grad_norm": 0.9700683418665813, "kl": 2.1544952392578125, "learning_rate": 1e-06, "loss": -0.2999, "step": 54 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1954.0, "completions/max_terminated_length": 1954.0, "completions/mean_length": 851.125, "completions/mean_terminated_length": 851.125, "completions/min_length": 9.0, "completions/min_terminated_length": 9.0, "epoch": 0.0050514327700220425, "grad_norm": 1.7938254223400716, "kl": 4.5559234619140625, "learning_rate": 1e-06, "loss": -0.2103, "num_tokens": 1338988.0, "reward": 0.2193053662776947, "reward_std": 0.07375349849462509, "rewards/format_reward_func": 0.8300000429153442, "rewards/lcs_reward_func": 0.05330537632107735, "step": 55 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "epoch": 0.005143277002204262, "grad_norm": 1.831632948103029, "kl": 4.60284423828125, "learning_rate": 1e-06, "loss": -0.2102, "step": 56 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1778.0, "completions/max_terminated_length": 1778.0, "completions/mean_length": 1001.375, "completions/mean_terminated_length": 1001.375, "completions/min_length": 9.0, "completions/min_terminated_length": 9.0, "epoch": 0.00523512123438648, "grad_norm": 2.3877586127598405, "kl": 4.5402374267578125, "learning_rate": 1e-06, "loss": -0.2191, "num_tokens": 1395263.0, "reward": 0.22619211673736572, "reward_std": 0.07270254194736481, "rewards/format_reward_func": 0.8700000047683716, "rewards/lcs_reward_func": 0.052192118018865585, "step": 57 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "epoch": 0.0053269654665687, "grad_norm": 2.1516149349049303, "kl": 4.290260314941406, "learning_rate": 1e-06, "loss": -0.2194, "step": 58 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 2000.0, "completions/max_terminated_length": 2000.0, "completions/mean_length": 1056.075, "completions/mean_terminated_length": 1056.075, "completions/min_length": 9.0, "completions/min_terminated_length": 9.0, "epoch": 0.005418809698750918, "grad_norm": 1.218185892788451, "kl": 1.4466400146484375, "learning_rate": 1e-06, "loss": -0.1457, "num_tokens": 1452716.0, "reward": 0.21461419761180878, "reward_std": 0.052187662571668625, "rewards/format_reward_func": 0.9099999666213989, "rewards/lcs_reward_func": 0.032614197582006454, "step": 59 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "epoch": 0.005510653930933138, "grad_norm": 1.2033731421993872, "kl": 1.3060531616210938, "learning_rate": 1e-06, "loss": -0.1458, "step": 60 }, { "epoch": 0.005510653930933138, "eval_completions/max_length": 1941, "eval_completions/mean_length": 1006.75, "eval_completions/min_length": 33, "eval_reward": 0.23812003433704376, "eval_reward_std": 0.06293292343616486, "step": 60 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 2013.0, "completions/max_terminated_length": 2013.0, "completions/mean_length": 1043.0, "completions/mean_terminated_length": 1043.0, "completions/min_length": 9.0, "completions/min_terminated_length": 9.0, "epoch": 0.005602498163115356, "grad_norm": 1.3991475236429036, "kl": 2.1066741943359375, "learning_rate": 1e-06, "loss": -0.1661, "num_tokens": 1506436.0, "reward": 0.2325197160243988, "reward_std": 0.07145832479000092, "rewards/format_reward_func": 0.8600000739097595, "rewards/lcs_reward_func": 0.06051970645785332, "step": 61 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "epoch": 0.005694342395297576, "grad_norm": 1.4079001953689518, "kl": 2.1243209838867188, "learning_rate": 1e-06, "loss": -0.1661, "step": 62 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1760.0, "completions/max_terminated_length": 1760.0, "completions/mean_length": 898.8, "completions/mean_terminated_length": 898.8, "completions/min_length": 9.0, "completions/min_terminated_length": 9.0, "epoch": 0.005786186627479794, "grad_norm": 1.0822550075119286, "kl": 3.12109375, "learning_rate": 1e-06, "loss": -0.3438, "num_tokens": 1556288.0, "reward": 0.2405313104391098, "reward_std": 0.09354240447282791, "rewards/format_reward_func": 0.8300000429153442, "rewards/lcs_reward_func": 0.07453129440546036, "step": 63 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "epoch": 0.005878030859662013, "grad_norm": 1.0584717578885592, "kl": 2.984375, "learning_rate": 1e-06, "loss": -0.344, "step": 64 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1431.0, "completions/max_terminated_length": 1431.0, "completions/mean_length": 741.8, "completions/mean_terminated_length": 741.8, "completions/min_length": 9.0, "completions/min_terminated_length": 9.0, "epoch": 0.005969875091844232, "grad_norm": 1.2789482228795541, "kl": 2.197265625, "learning_rate": 1e-06, "loss": -0.2848, "num_tokens": 1599990.0, "reward": 0.21671777963638306, "reward_std": 0.07907099276781082, "rewards/format_reward_func": 0.8600000739097595, "rewards/lcs_reward_func": 0.04471778869628906, "step": 65 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "epoch": 0.006061719324026451, "grad_norm": 1.266764856520884, "kl": 2.224609375, "learning_rate": 1e-06, "loss": -0.2848, "step": 66 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1968.0, "completions/max_terminated_length": 1968.0, "completions/mean_length": 1052.7, "completions/mean_terminated_length": 1052.7, "completions/min_length": 9.0, "completions/min_terminated_length": 9.0, "epoch": 0.00615356355620867, "grad_norm": 1.2688600523508209, "kl": 1.39453125, "learning_rate": 1e-06, "loss": -0.2192, "num_tokens": 1654308.0, "reward": 0.23257216811180115, "reward_std": 0.06556694209575653, "rewards/format_reward_func": 0.9000000357627869, "rewards/lcs_reward_func": 0.05257216840982437, "step": 67 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "epoch": 0.006245407788390889, "grad_norm": 1.2695224583429259, "kl": 1.43560791015625, "learning_rate": 1e-06, "loss": -0.2192, "step": 68 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1798.0, "completions/max_terminated_length": 1798.0, "completions/mean_length": 1001.15, "completions/mean_terminated_length": 1001.15, "completions/min_length": 9.0, "completions/min_terminated_length": 9.0, "epoch": 0.006337252020573108, "grad_norm": 1.4279803413786, "kl": 0.4173736572265625, "learning_rate": 1e-06, "loss": 0.0345, "num_tokens": 1707374.0, "reward": 0.2260584533214569, "reward_std": 0.04003484547138214, "rewards/format_reward_func": 0.9399999976158142, "rewards/lcs_reward_func": 0.03805844858288765, "step": 69 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "epoch": 0.006429096252755327, "grad_norm": 1.4245546877421398, "kl": 0.42337799072265625, "learning_rate": 1e-06, "loss": 0.0345, "step": 70 }, { "epoch": 0.006429096252755327, "eval_completions/max_length": 2045, "eval_completions/mean_length": 940.09375, "eval_completions/min_length": 33, "eval_reward": 0.21806684136390686, "eval_reward_std": 0.07097575068473816, "step": 70 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1891.0, "completions/max_terminated_length": 1891.0, "completions/mean_length": 833.775, "completions/mean_terminated_length": 833.775, "completions/min_length": 9.0, "completions/min_terminated_length": 9.0, "epoch": 0.006520940484937546, "grad_norm": 1.532083612743953, "kl": 2.4554443359375, "learning_rate": 1e-06, "loss": -0.103, "num_tokens": 1752925.0, "reward": 0.2318977415561676, "reward_std": 0.07339000701904297, "rewards/format_reward_func": 0.8600000739097595, "rewards/lcs_reward_func": 0.05989774689078331, "step": 71 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "epoch": 0.006612784717119765, "grad_norm": 1.5458953560334885, "kl": 2.7545013427734375, "learning_rate": 1e-06, "loss": -0.1027, "step": 72 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 2015.0, "completions/max_terminated_length": 2015.0, "completions/mean_length": 955.875, "completions/mean_terminated_length": 955.875, "completions/min_length": 9.0, "completions/min_terminated_length": 9.0, "epoch": 0.006704628949301984, "grad_norm": 1.5966261563593132, "kl": 2.0414581298828125, "learning_rate": 1e-06, "loss": -0.1909, "num_tokens": 1805010.0, "reward": 0.21281859278678894, "reward_std": 0.05903353542089462, "rewards/format_reward_func": 0.9000000357627869, "rewards/lcs_reward_func": 0.03281859681010246, "step": 73 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "epoch": 0.006796473181484203, "grad_norm": 1.5913807529250394, "kl": 2.0258331298828125, "learning_rate": 1e-06, "loss": -0.1909, "step": 74 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1949.0, "completions/max_terminated_length": 1949.0, "completions/mean_length": 847.3, "completions/mean_terminated_length": 847.3, "completions/min_length": 9.0, "completions/min_terminated_length": 9.0, "epoch": 0.006888317413666422, "grad_norm": 1.8091604327594668, "kl": 1.5551300048828125, "learning_rate": 1e-06, "loss": -0.1276, "num_tokens": 1850422.0, "reward": 0.22308267652988434, "reward_std": 0.05692993104457855, "rewards/format_reward_func": 0.9099999666213989, "rewards/lcs_reward_func": 0.041082676500082016, "step": 75 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "epoch": 0.0069801616458486405, "grad_norm": 1.793313998835583, "kl": 1.5044097900390625, "learning_rate": 1e-06, "loss": -0.1276, "step": 76 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1626.0, "completions/max_terminated_length": 1626.0, "completions/mean_length": 817.675, "completions/mean_terminated_length": 817.675, "completions/min_length": 9.0, "completions/min_terminated_length": 9.0, "epoch": 0.00707200587803086, "grad_norm": 1.4652517264367153, "kl": 1.5932769775390625, "learning_rate": 1e-06, "loss": -0.1444, "num_tokens": 1893259.0, "reward": 0.2210662066936493, "reward_std": 0.048665329813957214, "rewards/format_reward_func": 0.9100000262260437, "rewards/lcs_reward_func": 0.039066214114427567, "step": 77 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "epoch": 0.0071638501102130785, "grad_norm": 1.4504944985152288, "kl": 1.54644775390625, "learning_rate": 1e-06, "loss": -0.1444, "step": 78 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 2016.0, "completions/max_terminated_length": 2016.0, "completions/mean_length": 1031.425, "completions/mean_terminated_length": 1031.425, "completions/min_length": 9.0, "completions/min_terminated_length": 9.0, "epoch": 0.007255694342395298, "grad_norm": 1.49379469044819, "kl": 0.9149017333984375, "learning_rate": 1e-06, "loss": -0.075, "num_tokens": 1949706.0, "reward": 0.21732431650161743, "reward_std": 0.04848937690258026, "rewards/format_reward_func": 0.9200000166893005, "rewards/lcs_reward_func": 0.03332432731986046, "step": 79 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "epoch": 0.0073475385745775165, "grad_norm": 1.4689419152987504, "kl": 0.8819580078125, "learning_rate": 1e-06, "loss": -0.075, "step": 80 }, { "epoch": 0.0073475385745775165, "eval_completions/max_length": 1773, "eval_completions/mean_length": 1051.78125, "eval_completions/min_length": 33, "eval_reward": 0.23151257634162903, "eval_reward_std": 0.045354004949331284, "step": 80 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1878.0, "completions/max_terminated_length": 1878.0, "completions/mean_length": 920.725, "completions/mean_terminated_length": 920.725, "completions/min_length": 9.0, "completions/min_terminated_length": 9.0, "epoch": 0.007439382806759736, "grad_norm": 1.6000042069486635, "kl": 0.6986236572265625, "learning_rate": 1e-06, "loss": 0.0355, "num_tokens": 1998625.0, "reward": 0.23952794075012207, "reward_std": 0.03578774258494377, "rewards/format_reward_func": 0.9699999690055847, "rewards/lcs_reward_func": 0.04552793875336647, "step": 81 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "epoch": 0.007531227038941954, "grad_norm": 1.6048928271218026, "kl": 0.7575225830078125, "learning_rate": 1e-06, "loss": 0.0355, "step": 82 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1929.0, "completions/max_terminated_length": 1929.0, "completions/mean_length": 1051.1, "completions/mean_terminated_length": 1051.1, "completions/min_length": 473.0, "completions/min_terminated_length": 473.0, "epoch": 0.007623071271124174, "grad_norm": 1.6622202652089961, "kl": 0.0103607177734375, "learning_rate": 1e-06, "loss": -0.0052, "num_tokens": 2054249.0, "reward": 0.2544408440589905, "reward_std": 0.02876886911690235, "rewards/format_reward_func": 0.9850000739097595, "rewards/lcs_reward_func": 0.057440828531980515, "step": 83 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "epoch": 0.007714915503306392, "grad_norm": 1.6711405480859438, "kl": 0.010833740234375, "learning_rate": 1e-06, "loss": -0.0052, "step": 84 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1665.0, "completions/max_terminated_length": 1665.0, "completions/mean_length": 863.25, "completions/mean_terminated_length": 863.25, "completions/min_length": 9.0, "completions/min_terminated_length": 9.0, "epoch": 0.007806759735488611, "grad_norm": 1.5596144146559612, "kl": 1.1883697509765625, "learning_rate": 1e-06, "loss": -0.0611, "num_tokens": 2101179.0, "reward": 0.2613794803619385, "reward_std": 0.06008349731564522, "rewards/format_reward_func": 0.9399999976158142, "rewards/lcs_reward_func": 0.07337947934865952, "step": 85 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "epoch": 0.00789860396767083, "grad_norm": 1.5643789434736712, "kl": 1.21990966796875, "learning_rate": 1e-06, "loss": -0.0611, "step": 86 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1878.0, "completions/max_terminated_length": 1878.0, "completions/mean_length": 874.4, "completions/mean_terminated_length": 874.4, "completions/min_length": 9.0, "completions/min_terminated_length": 9.0, "epoch": 0.00799044819985305, "grad_norm": 1.7164853856370337, "kl": 2.06787109375, "learning_rate": 1e-06, "loss": -0.1248, "num_tokens": 2150155.0, "reward": 0.2113932967185974, "reward_std": 0.0450090691447258, "rewards/format_reward_func": 0.8949999809265137, "rewards/lcs_reward_func": 0.03239328786730766, "step": 87 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "epoch": 0.008082292432035268, "grad_norm": 1.6836941181967466, "kl": 1.8844757080078125, "learning_rate": 1e-06, "loss": -0.125, "step": 88 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1801.0, "completions/max_terminated_length": 1801.0, "completions/mean_length": 856.1, "completions/mean_terminated_length": 856.1, "completions/min_length": 9.0, "completions/min_terminated_length": 9.0, "epoch": 0.008174136664217487, "grad_norm": 1.6060443906303505, "kl": 0.6384735107421875, "learning_rate": 1e-06, "loss": 0.0493, "num_tokens": 2198719.0, "reward": 0.23078730702400208, "reward_std": 0.030345093458890915, "rewards/format_reward_func": 0.9699999690055847, "rewards/lcs_reward_func": 0.036787305027246475, "step": 89 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "epoch": 0.008265980896399705, "grad_norm": 1.6107603772408245, "kl": 0.6232452392578125, "learning_rate": 1e-06, "loss": 0.0493, "step": 90 }, { "epoch": 0.008265980896399705, "eval_completions/max_length": 1363, "eval_completions/mean_length": 814.8125, "eval_completions/min_length": 33, "eval_reward": 0.24562440812587738, "eval_reward_std": 0.07713833451271057, "step": 90 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1551.0, "completions/max_terminated_length": 1551.0, "completions/mean_length": 899.9, "completions/mean_terminated_length": 899.9, "completions/min_length": 9.0, "completions/min_terminated_length": 9.0, "epoch": 0.008357825128581926, "grad_norm": 1.66604613571463, "kl": 0.5090789794921875, "learning_rate": 1e-06, "loss": 0.0378, "num_tokens": 2245695.0, "reward": 0.24726460874080658, "reward_std": 0.036231789737939835, "rewards/format_reward_func": 0.9699999690055847, "rewards/lcs_reward_func": 0.05326459929347038, "step": 91 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "epoch": 0.008449669360764144, "grad_norm": 1.6507974800991143, "kl": 0.517242431640625, "learning_rate": 1e-06, "loss": 0.0378, "step": 92 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1269.0, "completions/max_terminated_length": 1269.0, "completions/mean_length": 749.15, "completions/mean_terminated_length": 749.15, "completions/min_length": 9.0, "completions/min_terminated_length": 9.0, "epoch": 0.008541513592946363, "grad_norm": 1.9377993694564883, "kl": 0.8455352783203125, "learning_rate": 1e-06, "loss": -0.0226, "num_tokens": 2286321.0, "reward": 0.2655041515827179, "reward_std": 0.06015206128358841, "rewards/format_reward_func": 0.9699999690055847, "rewards/lcs_reward_func": 0.0715041384100914, "step": 93 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "epoch": 0.008633357825128581, "grad_norm": 1.9289812213800934, "kl": 0.7676544189453125, "learning_rate": 1e-06, "loss": -0.0227, "step": 94 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1792.0, "completions/max_terminated_length": 1792.0, "completions/mean_length": 887.275, "completions/mean_terminated_length": 887.275, "completions/min_length": 9.0, "completions/min_terminated_length": 9.0, "epoch": 0.008725202057310802, "grad_norm": 1.5084325888205004, "kl": 1.1592254638671875, "learning_rate": 1e-06, "loss": -0.034, "num_tokens": 2333032.0, "reward": 0.29208117723464966, "reward_std": 0.07356677949428558, "rewards/format_reward_func": 0.9449999928474426, "rewards/lcs_reward_func": 0.10308118164539337, "step": 95 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "epoch": 0.00881704628949302, "grad_norm": 1.5038288881177668, "kl": 1.1751251220703125, "learning_rate": 1e-06, "loss": -0.0339, "step": 96 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1859.0, "completions/max_terminated_length": 1859.0, "completions/mean_length": 905.975, "completions/mean_terminated_length": 905.975, "completions/min_length": 9.0, "completions/min_terminated_length": 9.0, "epoch": 0.008908890521675239, "grad_norm": 1.6650831756792426, "kl": 1.7579498291015625, "learning_rate": 1e-06, "loss": -0.1085, "num_tokens": 2385061.0, "reward": 0.24999132752418518, "reward_std": 0.0621689110994339, "rewards/format_reward_func": 0.9149999618530273, "rewards/lcs_reward_func": 0.06699130684137344, "step": 97 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "epoch": 0.009000734753857457, "grad_norm": 1.6701005438071024, "kl": 1.7736968994140625, "learning_rate": 1e-06, "loss": -0.1085, "step": 98 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1999.0, "completions/max_terminated_length": 1999.0, "completions/mean_length": 931.1, "completions/mean_terminated_length": 931.1, "completions/min_length": 9.0, "completions/min_terminated_length": 9.0, "epoch": 0.009092578986039677, "grad_norm": 1.1066387280360448, "kl": 1.6706085205078125, "learning_rate": 1e-06, "loss": -0.1735, "num_tokens": 2433405.0, "reward": 0.2186073362827301, "reward_std": 0.05433933064341545, "rewards/format_reward_func": 0.9300000071525574, "rewards/lcs_reward_func": 0.03260732442140579, "step": 99 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "epoch": 0.009184423218221896, "grad_norm": 1.1270276768467666, "kl": 1.70977783203125, "learning_rate": 1e-06, "loss": -0.1734, "step": 100 }, { "epoch": 0.009184423218221896, "eval_completions/max_length": 2003, "eval_completions/mean_length": 947.28125, "eval_completions/min_length": 476, "eval_reward": 0.24798785150051117, "eval_reward_std": 0.04357936605811119, "step": 100 } ], "logging_steps": 1, "max_steps": 100, "num_input_tokens_seen": 2433405, "num_train_epochs": 1, "save_steps": 500, "stateful_callbacks": { "TrainerControl": { "args": { "should_epoch_stop": false, "should_evaluate": true, "should_log": false, "should_save": true, "should_training_stop": true }, "attributes": {} } }, "total_flos": 0.0, "train_batch_size": 10, "trial_name": null, "trial_params": null }