diff --git "a/trainer_state.json" "b/trainer_state.json" new file mode 100644--- /dev/null +++ "b/trainer_state.json" @@ -0,0 +1,2911 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 0.9997795900374697, + "eval_steps": 500, + "global_step": 567, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "clip_ratio": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 348.9375, + "completions/max_terminated_length": 348.9375, + "completions/mean_length": 166.421875, + "completions/mean_terminated_length": 166.421875, + "completions/min_length": 46.9375, + "completions/min_terminated_length": 46.9375, + "epoch": 0.001763279700242451, + "grad_norm": 0.9710755373974377, + "kl": 0.0, + "learning_rate": 0.0, + "loss": -0.1942, + "num_tokens": 147932.0, + "reward": 0.7169940862804651, + "reward_std": 0.2023643054999411, + "rewards/format_reward/mean": 0.6015625, + "rewards/format_reward/std": 0.4860950894653797, + "rewards/qatch_metrics/mean": 0.7311028698459268, + "rewards/qatch_metrics/std": 0.21332682901993394, + "rewards/tag_count_reward/mean": 0.7080078125, + "rewards/tag_count_reward/std": 0.3562574004754424, + "step": 1 + }, + { + "clip_ratio": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 320.859375, + "completions/max_terminated_length": 320.859375, + "completions/mean_length": 157.0234375, + "completions/mean_terminated_length": 157.0234375, + "completions/min_length": 45.984375, + "completions/min_terminated_length": 45.984375, + "epoch": 0.008816398501212255, + "grad_norm": 1.3231567694690758, + "kl": 0.000191517174243927, + "learning_rate": 7.017543859649122e-08, + "loss": -0.3181, + "num_tokens": 670628.0, + "reward": 0.7213839697651565, + "reward_std": 0.15228567429585382, + "rewards/format_reward/mean": 0.5888671875, + "rewards/format_reward/std": 0.4847157197073102, + "rewards/qatch_metrics/mean": 0.7386370480526239, + "rewards/qatch_metrics/std": 0.13745591492079257, + "rewards/tag_count_reward/mean": 0.693115234375, + "rewards/tag_count_reward/std": 0.36212220159359276, + "step": 5 + }, + { + "clip_ratio": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 336.6, + "completions/max_terminated_length": 336.6, + "completions/mean_length": 168.38125, + "completions/mean_terminated_length": 168.38125, + "completions/min_length": 47.6625, + "completions/min_terminated_length": 47.6625, + "epoch": 0.01763279700242451, + "grad_norm": 1.2344979720798859, + "kl": 0.0002501368522644043, + "learning_rate": 1.5789473684210525e-07, + "loss": -0.2443, + "num_tokens": 1345916.0, + "reward": 0.6422883274964988, + "reward_std": 0.19879084336571395, + "rewards/format_reward/mean": 0.596875, + "rewards/format_reward/std": 0.4875950779765844, + "rewards/qatch_metrics/mean": 0.6440179711673408, + "rewards/qatch_metrics/std": 0.19703516885638236, + "rewards/tag_count_reward/mean": 0.7037109375, + "rewards/tag_count_reward/std": 0.3609486103057861, + "step": 10 + }, + { + "clip_ratio": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 334.6375, + "completions/max_terminated_length": 334.6375, + "completions/mean_length": 172.48515625, + "completions/mean_terminated_length": 172.48515625, + "completions/min_length": 47.9625, + "completions/min_terminated_length": 47.9625, + "epoch": 0.026449195503636766, + "grad_norm": 1.0324306738907167, + "kl": 0.0003110170364379883, + "learning_rate": 2.456140350877193e-07, + "loss": -0.2119, + "num_tokens": 2061897.0, + "reward": 0.7045063060708344, + "reward_std": 0.17503634537570179, + "rewards/format_reward/mean": 0.6765625, + "rewards/format_reward/std": 0.4584621708840132, + "rewards/qatch_metrics/mean": 0.7045317724347114, + "rewards/qatch_metrics/std": 0.17522094006126282, + "rewards/tag_count_reward/mean": 0.7599609375, + "rewards/tag_count_reward/std": 0.34046435691416266, + "step": 15 + }, + { + "clip_ratio": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 359.0375, + "completions/max_terminated_length": 359.0375, + "completions/mean_length": 213.50859375, + "completions/mean_terminated_length": 213.50859375, + "completions/min_length": 63.2125, + "completions/min_terminated_length": 63.2125, + "epoch": 0.03526559400484902, + "grad_norm": 0.7271916297615016, + "kl": 0.0008575439453125, + "learning_rate": 3.333333333333333e-07, + "loss": -0.1464, + "num_tokens": 2779524.0, + "reward": 0.736725780274719, + "reward_std": 0.14974234900437294, + "rewards/format_reward/mean": 0.82265625, + "rewards/format_reward/std": 0.35349783338606355, + "rewards/qatch_metrics/mean": 0.7184940161881969, + "rewards/qatch_metrics/std": 0.14797394773922862, + "rewards/tag_count_reward/mean": 0.8748046875, + "rewards/tag_count_reward/std": 0.2501601692289114, + "step": 20 + }, + { + "clip_ratio": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 345.9, + "completions/max_terminated_length": 345.9, + "completions/mean_length": 221.61953125, + "completions/mean_terminated_length": 221.61953125, + "completions/min_length": 110.9625, + "completions/min_terminated_length": 110.9625, + "epoch": 0.04408199250606128, + "grad_norm": 0.36089994298099687, + "kl": 0.002716684341430664, + "learning_rate": 4.2105263157894733e-07, + "loss": -0.0294, + "num_tokens": 3532397.0, + "reward": 0.7203821750357747, + "reward_std": 0.15227624527178704, + "rewards/format_reward/mean": 0.94375, + "rewards/format_reward/std": 0.16674657054245473, + "rewards/qatch_metrics/mean": 0.6798958362895065, + "rewards/qatch_metrics/std": 0.1698521633632481, + "rewards/tag_count_reward/mean": 0.9619140625, + "rewards/tag_count_reward/std": 0.11083460114896297, + "step": 25 + }, + { + "clip_ratio": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 351.1625, + "completions/max_terminated_length": 351.1625, + "completions/mean_length": 233.1609375, + "completions/mean_terminated_length": 233.1609375, + "completions/min_length": 146.275, + "completions/min_terminated_length": 146.275, + "epoch": 0.05289839100727353, + "grad_norm": 0.39496099506167615, + "kl": 0.002033853530883789, + "learning_rate": 5.087719298245614e-07, + "loss": 0.0164, + "num_tokens": 4292411.0, + "reward": 0.7153215611353516, + "reward_std": 0.15457577785709872, + "rewards/format_reward/mean": 0.98828125, + "rewards/format_reward/std": 0.046875, + "rewards/qatch_metrics/mean": 0.6668190175667406, + "rewards/qatch_metrics/std": 0.1794413580093533, + "rewards/tag_count_reward/mean": 0.9939453125, + "rewards/tag_count_reward/std": 0.02421875, + "step": 30 + }, + { + "clip_ratio": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 357.1375, + "completions/max_terminated_length": 357.1375, + "completions/mean_length": 236.98046875, + "completions/mean_terminated_length": 236.98046875, + "completions/min_length": 144.2375, + "completions/min_terminated_length": 144.2375, + "epoch": 0.06171478950848578, + "grad_norm": 0.40335285901379214, + "kl": 0.004096126556396485, + "learning_rate": 5.964912280701754e-07, + "loss": -0.0084, + "num_tokens": 5094210.0, + "reward": 0.7295356256887316, + "reward_std": 0.15927550423293724, + "rewards/format_reward/mean": 0.99296875, + "rewards/format_reward/std": 0.028125, + "rewards/qatch_metrics/mean": 0.6829440153203905, + "rewards/qatch_metrics/std": 0.1853304866242979, + "rewards/tag_count_reward/mean": 0.9947265625, + "rewards/tag_count_reward/std": 0.02109375, + "step": 35 + }, + { + "clip_ratio": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 348.875, + "completions/max_terminated_length": 348.875, + "completions/mean_length": 225.73203125, + "completions/mean_terminated_length": 225.73203125, + "completions/min_length": 141.75, + "completions/min_terminated_length": 141.75, + "epoch": 0.07053118800969804, + "grad_norm": 0.28998715845893785, + "kl": 0.001429271697998047, + "learning_rate": 6.842105263157895e-07, + "loss": 0.0087, + "num_tokens": 5833499.0, + "reward": 0.7490684226155281, + "reward_std": 0.12702710015701085, + "rewards/format_reward/mean": 0.9984375, + "rewards/format_reward/std": 0.00625, + "rewards/qatch_metrics/mean": 0.7050161464139819, + "rewards/qatch_metrics/std": 0.14899895801208912, + "rewards/tag_count_reward/mean": 0.99921875, + "rewards/tag_count_reward/std": 0.003125, + "step": 40 + }, + { + "clip_ratio": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 337.7625, + "completions/max_terminated_length": 337.7625, + "completions/mean_length": 234.1765625, + "completions/mean_terminated_length": 234.1765625, + "completions/min_length": 149.025, + "completions/min_terminated_length": 149.025, + "epoch": 0.0793475865109103, + "grad_norm": 0.3515825769486025, + "kl": 0.001779651641845703, + "learning_rate": 7.719298245614034e-07, + "loss": 0.0103, + "num_tokens": 6625773.0, + "reward": 0.7813593098893762, + "reward_std": 0.1548473397095222, + "rewards/format_reward/mean": 0.99765625, + "rewards/format_reward/std": 0.009375, + "rewards/qatch_metrics/mean": 0.7431203166022897, + "rewards/qatch_metrics/std": 0.18166826255619525, + "rewards/tag_count_reward/mean": 0.998828125, + "rewards/tag_count_reward/std": 0.0046875, + "step": 45 + }, + { + "clip_ratio": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 345.325, + "completions/max_terminated_length": 345.325, + "completions/mean_length": 230.6203125, + "completions/mean_terminated_length": 230.6203125, + "completions/min_length": 150.1625, + "completions/min_terminated_length": 150.1625, + "epoch": 0.08816398501212255, + "grad_norm": 0.30438917372160834, + "kl": 0.0018757820129394532, + "learning_rate": 8.596491228070175e-07, + "loss": 0.0006, + "num_tokens": 7364007.0, + "reward": 0.7656656216830016, + "reward_std": 0.1071579195689992, + "rewards/format_reward/mean": 0.99921875, + "rewards/format_reward/std": 0.003125, + "rewards/qatch_metrics/mean": 0.724415887008945, + "rewards/qatch_metrics/std": 0.12601496450661215, + "rewards/tag_count_reward/mean": 0.9998046875, + "rewards/tag_count_reward/std": 0.00078125, + "step": 50 + }, + { + "clip_ratio": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 340.725, + "completions/max_terminated_length": 340.725, + "completions/mean_length": 219.0265625, + "completions/mean_terminated_length": 219.0265625, + "completions/min_length": 140.775, + "completions/min_terminated_length": 140.775, + "epoch": 0.09698038351333481, + "grad_norm": 0.32051285299560317, + "kl": 0.002438640594482422, + "learning_rate": 9.473684210526315e-07, + "loss": 0.0062, + "num_tokens": 8093081.0, + "reward": 0.7895074604079128, + "reward_std": 0.10295408805832267, + "rewards/format_reward/mean": 0.99921875, + "rewards/format_reward/std": 0.003125, + "rewards/qatch_metrics/mean": 0.7524651106446981, + "rewards/qatch_metrics/std": 0.12109476723708212, + "rewards/tag_count_reward/mean": 0.9998046875, + "rewards/tag_count_reward/std": 0.00078125, + "step": 55 + }, + { + "clip_ratio": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 333.6125, + "completions/max_terminated_length": 333.6125, + "completions/mean_length": 214.33203125, + "completions/mean_terminated_length": 214.33203125, + "completions/min_length": 138.125, + "completions/min_terminated_length": 138.125, + "epoch": 0.10579678201454706, + "grad_norm": 0.3687506384245934, + "kl": 0.0034315109252929686, + "learning_rate": 1e-06, + "loss": 0.0167, + "num_tokens": 8839074.0, + "reward": 0.7963492956012488, + "reward_std": 0.13961084922775627, + "rewards/format_reward/mean": 0.99921875, + "rewards/format_reward/std": 0.003125, + "rewards/qatch_metrics/mean": 0.76051432879176, + "rewards/qatch_metrics/std": 0.16409257110208272, + "rewards/tag_count_reward/mean": 0.9998046875, + "rewards/tag_count_reward/std": 0.00078125, + "step": 60 + }, + { + "clip_ratio": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 314.45, + "completions/max_terminated_length": 314.45, + "completions/mean_length": 205.66484375, + "completions/mean_terminated_length": 205.66484375, + "completions/min_length": 138.325, + "completions/min_terminated_length": 138.325, + "epoch": 0.11461318051575932, + "grad_norm": 0.3598691865301115, + "kl": 0.004790306091308594, + "learning_rate": 1e-06, + "loss": -0.0072, + "num_tokens": 9592197.0, + "reward": 0.7755136819556355, + "reward_std": 0.12071140363659652, + "rewards/format_reward/mean": 1.0, + "rewards/format_reward/std": 0.0, + "rewards/qatch_metrics/mean": 0.735898441291647, + "rewards/qatch_metrics/std": 0.14201342400228895, + "rewards/tag_count_reward/mean": 1.0, + "rewards/tag_count_reward/std": 0.0, + "step": 65 + }, + { + "clip_ratio": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 329.4875, + "completions/max_terminated_length": 329.4875, + "completions/mean_length": 218.82734375, + "completions/mean_terminated_length": 218.82734375, + "completions/min_length": 144.225, + "completions/min_terminated_length": 144.225, + "epoch": 0.12342957901697156, + "grad_norm": 0.40718780074992844, + "kl": 0.005454826354980469, + "learning_rate": 1e-06, + "loss": 0.0052, + "num_tokens": 10323832.0, + "reward": 0.8082829523831606, + "reward_std": 0.12197014213888906, + "rewards/format_reward/mean": 1.0, + "rewards/format_reward/std": 0.0, + "rewards/qatch_metrics/mean": 0.7744505235925316, + "rewards/qatch_metrics/std": 0.1434942939085886, + "rewards/tag_count_reward/mean": 1.0, + "rewards/tag_count_reward/std": 0.0, + "step": 70 + }, + { + "clip_ratio": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 336.075, + "completions/max_terminated_length": 336.075, + "completions/mean_length": 220.25234375, + "completions/mean_terminated_length": 220.25234375, + "completions/min_length": 142.125, + "completions/min_terminated_length": 142.125, + "epoch": 0.13224597751818382, + "grad_norm": 0.4066036566641524, + "kl": 0.005206871032714844, + "learning_rate": 1e-06, + "loss": 0.003, + "num_tokens": 11058347.0, + "reward": 0.7646768478676677, + "reward_std": 0.15216110937763005, + "rewards/format_reward/mean": 1.0, + "rewards/format_reward/std": 0.0, + "rewards/qatch_metrics/mean": 0.7231492260238156, + "rewards/qatch_metrics/std": 0.17901307856664062, + "rewards/tag_count_reward/mean": 1.0, + "rewards/tag_count_reward/std": 0.0, + "step": 75 + }, + { + "clip_ratio": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 323.4625, + "completions/max_terminated_length": 323.4625, + "completions/mean_length": 212.896875, + "completions/mean_terminated_length": 212.896875, + "completions/min_length": 138.65, + "completions/min_terminated_length": 138.65, + "epoch": 0.14106237601939609, + "grad_norm": 0.3757999153884869, + "kl": 0.006637382507324219, + "learning_rate": 1e-06, + "loss": 0.0123, + "num_tokens": 11818007.0, + "reward": 0.7600463172420859, + "reward_std": 0.11857278576935641, + "rewards/format_reward/mean": 1.0, + "rewards/format_reward/std": 0.0, + "rewards/qatch_metrics/mean": 0.7177130273543298, + "rewards/qatch_metrics/std": 0.1394514435902238, + "rewards/tag_count_reward/mean": 0.9998046875, + "rewards/tag_count_reward/std": 0.00078125, + "step": 80 + }, + { + "clip_ratio": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 318.2375, + "completions/max_terminated_length": 318.2375, + "completions/mean_length": 212.54140625, + "completions/mean_terminated_length": 212.54140625, + "completions/min_length": 140.9125, + "completions/min_terminated_length": 140.9125, + "epoch": 0.14987877452060833, + "grad_norm": 0.2843877461846444, + "kl": 0.006991195678710938, + "learning_rate": 1e-06, + "loss": 0.0103, + "num_tokens": 12562892.0, + "reward": 0.7912707846611738, + "reward_std": 0.12222371264360846, + "rewards/format_reward/mean": 1.0, + "rewards/format_reward/std": 0.0, + "rewards/qatch_metrics/mean": 0.7544361999258399, + "rewards/qatch_metrics/std": 0.14379260735586286, + "rewards/tag_count_reward/mean": 1.0, + "rewards/tag_count_reward/std": 0.0, + "step": 85 + }, + { + "clip_ratio": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 316.575, + "completions/max_terminated_length": 316.575, + "completions/mean_length": 211.7421875, + "completions/mean_terminated_length": 211.7421875, + "completions/min_length": 143.1125, + "completions/min_terminated_length": 143.1125, + "epoch": 0.1586951730218206, + "grad_norm": 0.33442141566841666, + "kl": 0.00792999267578125, + "learning_rate": 1e-06, + "loss": 0.001, + "num_tokens": 13308402.0, + "reward": 0.8190947765484452, + "reward_std": 0.12180806350661441, + "rewards/format_reward/mean": 1.0, + "rewards/format_reward/std": 0.0, + "rewards/qatch_metrics/mean": 0.7871703177705058, + "rewards/qatch_metrics/std": 0.14330361068132333, + "rewards/tag_count_reward/mean": 1.0, + "rewards/tag_count_reward/std": 0.0, + "step": 90 + }, + { + "clip_ratio": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 327.6, + "completions/max_terminated_length": 327.6, + "completions/mean_length": 220.39140625, + "completions/mean_terminated_length": 220.39140625, + "completions/min_length": 144.9375, + "completions/min_terminated_length": 144.9375, + "epoch": 0.16751157152303284, + "grad_norm": 0.2060349260059472, + "kl": 0.007589149475097656, + "learning_rate": 1e-06, + "loss": 0.0053, + "num_tokens": 14090727.0, + "reward": 0.8084983274340629, + "reward_std": 0.09074296336621046, + "rewards/format_reward/mean": 1.0, + "rewards/format_reward/std": 0.0, + "rewards/qatch_metrics/mean": 0.774703911319375, + "rewards/qatch_metrics/std": 0.1067564318422228, + "rewards/tag_count_reward/mean": 1.0, + "rewards/tag_count_reward/std": 0.0, + "step": 95 + }, + { + "clip_ratio": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 345.4375, + "completions/max_terminated_length": 345.4375, + "completions/mean_length": 233.321875, + "completions/mean_terminated_length": 233.321875, + "completions/min_length": 150.725, + "completions/min_terminated_length": 150.725, + "epoch": 0.1763279700242451, + "grad_norm": 0.3437681507721623, + "kl": 0.008571624755859375, + "learning_rate": 1e-06, + "loss": 0.0018, + "num_tokens": 14870899.0, + "reward": 0.7850898955017328, + "reward_std": 0.11421530576772057, + "rewards/format_reward/mean": 0.99921875, + "rewards/format_reward/std": 0.003125, + "rewards/qatch_metrics/mean": 0.747267971560359, + "rewards/qatch_metrics/std": 0.13395735200028866, + "rewards/tag_count_reward/mean": 0.9998046875, + "rewards/tag_count_reward/std": 0.00078125, + "step": 100 + }, + { + "clip_ratio": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 328.7, + "completions/max_terminated_length": 328.7, + "completions/mean_length": 218.9609375, + "completions/mean_terminated_length": 218.9609375, + "completions/min_length": 147.7125, + "completions/min_terminated_length": 147.7125, + "epoch": 0.18514436852545735, + "grad_norm": 0.2956106829157968, + "kl": 0.0087432861328125, + "learning_rate": 1e-06, + "loss": 0.0023, + "num_tokens": 15637185.0, + "reward": 0.8008848559111357, + "reward_std": 0.1065025228075683, + "rewards/format_reward/mean": 1.0, + "rewards/format_reward/std": 0.0, + "rewards/qatch_metrics/mean": 0.7657468795776368, + "rewards/qatch_metrics/std": 0.12529709176160395, + "rewards/tag_count_reward/mean": 1.0, + "rewards/tag_count_reward/std": 0.0, + "step": 105 + }, + { + "clip_ratio": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 354.7, + "completions/max_terminated_length": 354.7, + "completions/mean_length": 241.70234375, + "completions/mean_terminated_length": 241.70234375, + "completions/min_length": 153.675, + "completions/min_terminated_length": 153.675, + "epoch": 0.19396076702666962, + "grad_norm": 0.3156816970551577, + "kl": 0.00846099853515625, + "learning_rate": 1e-06, + "loss": -0.0043, + "num_tokens": 16427236.0, + "reward": 0.8307245042175054, + "reward_std": 0.12043557950855757, + "rewards/format_reward/mean": 1.0, + "rewards/format_reward/std": 0.0, + "rewards/qatch_metrics/mean": 0.8008523487951607, + "rewards/qatch_metrics/std": 0.14168892623565627, + "rewards/tag_count_reward/mean": 1.0, + "rewards/tag_count_reward/std": 0.0, + "step": 110 + }, + { + "clip_ratio": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 369.025, + "completions/max_terminated_length": 369.025, + "completions/mean_length": 242.18203125, + "completions/mean_terminated_length": 242.18203125, + "completions/min_length": 155.2875, + "completions/min_terminated_length": 155.2875, + "epoch": 0.20277716552788186, + "grad_norm": 0.35044764518300353, + "kl": 0.008629226684570312, + "learning_rate": 1e-06, + "loss": 0.0172, + "num_tokens": 17204669.0, + "reward": 0.7873043244704604, + "reward_std": 0.11700194676523097, + "rewards/format_reward/mean": 0.99921875, + "rewards/format_reward/std": 0.003125, + "rewards/qatch_metrics/mean": 0.7498731818312081, + "rewards/qatch_metrics/std": 0.13737923657754436, + "rewards/tag_count_reward/mean": 0.9998046875, + "rewards/tag_count_reward/std": 0.00078125, + "step": 115 + }, + { + "clip_ratio": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 326.7, + "completions/max_terminated_length": 326.7, + "completions/mean_length": 223.1546875, + "completions/mean_terminated_length": 223.1546875, + "completions/min_length": 145.8125, + "completions/min_terminated_length": 145.8125, + "epoch": 0.21159356402909413, + "grad_norm": 0.256901537727429, + "kl": 0.008853530883789063, + "learning_rate": 1e-06, + "loss": 0.0019, + "num_tokens": 17978371.0, + "reward": 0.8254757545888424, + "reward_std": 0.09378883789759129, + "rewards/format_reward/mean": 1.0, + "rewards/format_reward/std": 0.0, + "rewards/qatch_metrics/mean": 0.794677343731746, + "rewards/qatch_metrics/std": 0.11033981533837504, + "rewards/tag_count_reward/mean": 1.0, + "rewards/tag_count_reward/std": 0.0, + "step": 120 + }, + { + "clip_ratio": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 320.0, + "completions/max_terminated_length": 320.0, + "completions/mean_length": 214.93828125, + "completions/mean_terminated_length": 214.93828125, + "completions/min_length": 144.85, + "completions/min_terminated_length": 144.85, + "epoch": 0.22040996253030637, + "grad_norm": 0.3734181940444686, + "kl": 0.009600830078125, + "learning_rate": 1e-06, + "loss": 0.0006, + "num_tokens": 18723748.0, + "reward": 0.7991562966257334, + "reward_std": 0.09936780408024788, + "rewards/format_reward/mean": 1.0, + "rewards/format_reward/std": 0.0, + "rewards/qatch_metrics/mean": 0.7637132841162384, + "rewards/qatch_metrics/std": 0.11690330407582224, + "rewards/tag_count_reward/mean": 1.0, + "rewards/tag_count_reward/std": 0.0, + "step": 125 + }, + { + "clip_ratio": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 328.3, + "completions/max_terminated_length": 328.3, + "completions/mean_length": 220.58515625, + "completions/mean_terminated_length": 220.58515625, + "completions/min_length": 137.9875, + "completions/min_terminated_length": 137.9875, + "epoch": 0.22922636103151864, + "grad_norm": 0.38207935979690344, + "kl": 0.010348129272460937, + "learning_rate": 1e-06, + "loss": 0.012, + "num_tokens": 19500401.0, + "reward": 0.812997136451304, + "reward_std": 0.12226196355186403, + "rewards/format_reward/mean": 1.0, + "rewards/format_reward/std": 0.0, + "rewards/qatch_metrics/mean": 0.7799966168124228, + "rewards/qatch_metrics/std": 0.14383760886266828, + "rewards/tag_count_reward/mean": 1.0, + "rewards/tag_count_reward/std": 0.0, + "step": 130 + }, + { + "clip_ratio": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 338.6125, + "completions/max_terminated_length": 338.6125, + "completions/mean_length": 220.6390625, + "completions/mean_terminated_length": 220.6390625, + "completions/min_length": 142.0375, + "completions/min_terminated_length": 142.0375, + "epoch": 0.23804275953273088, + "grad_norm": 0.3417278304849786, + "kl": 0.011602020263671875, + "learning_rate": 1e-06, + "loss": 0.0024, + "num_tokens": 20253139.0, + "reward": 0.7694992732256651, + "reward_std": 0.10531347445212305, + "rewards/format_reward/mean": 1.0, + "rewards/format_reward/std": 0.0, + "rewards/qatch_metrics/mean": 0.7288226603297516, + "rewards/qatch_metrics/std": 0.12389820874668658, + "rewards/tag_count_reward/mean": 1.0, + "rewards/tag_count_reward/std": 0.0, + "step": 135 + }, + { + "clip_ratio": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 312.3, + "completions/max_terminated_length": 312.3, + "completions/mean_length": 208.0296875, + "completions/mean_terminated_length": 208.0296875, + "completions/min_length": 136.55, + "completions/min_terminated_length": 136.55, + "epoch": 0.24685915803394312, + "grad_norm": 0.3425773253691935, + "kl": 0.010589218139648438, + "learning_rate": 1e-06, + "loss": 0.0116, + "num_tokens": 20952281.0, + "reward": 0.8347327882423997, + "reward_std": 0.10723668891005218, + "rewards/format_reward/mean": 1.0, + "rewards/format_reward/std": 0.0, + "rewards/qatch_metrics/mean": 0.8055679749464616, + "rewards/qatch_metrics/std": 0.1261608180589974, + "rewards/tag_count_reward/mean": 1.0, + "rewards/tag_count_reward/std": 0.0, + "step": 140 + }, + { + "clip_ratio": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 336.2375, + "completions/max_terminated_length": 336.2375, + "completions/mean_length": 226.9140625, + "completions/mean_terminated_length": 226.9140625, + "completions/min_length": 141.725, + "completions/min_terminated_length": 141.725, + "epoch": 0.2556755565351554, + "grad_norm": 0.3141154785420489, + "kl": 0.011150741577148437, + "learning_rate": 1e-06, + "loss": 0.0045, + "num_tokens": 21701563.0, + "reward": 0.8115479167550802, + "reward_std": 0.11712859515100718, + "rewards/format_reward/mean": 0.99921875, + "rewards/format_reward/std": 0.003125, + "rewards/qatch_metrics/mean": 0.778395052952692, + "rewards/qatch_metrics/std": 0.13778491392731668, + "rewards/tag_count_reward/mean": 0.9998046875, + "rewards/tag_count_reward/std": 0.00078125, + "step": 145 + }, + { + "clip_ratio": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 324.325, + "completions/max_terminated_length": 324.325, + "completions/mean_length": 214.4046875, + "completions/mean_terminated_length": 214.4046875, + "completions/min_length": 137.175, + "completions/min_terminated_length": 137.175, + "epoch": 0.26449195503636763, + "grad_norm": 0.346933040349207, + "kl": 0.020714950561523438, + "learning_rate": 1e-06, + "loss": 0.0045, + "num_tokens": 22407937.0, + "reward": 0.8108294125646353, + "reward_std": 0.08920670928346226, + "rewards/format_reward/mean": 1.0, + "rewards/format_reward/std": 0.0, + "rewards/qatch_metrics/mean": 0.7774463596753776, + "rewards/qatch_metrics/std": 0.1049490759614855, + "rewards/tag_count_reward/mean": 1.0, + "rewards/tag_count_reward/std": 0.0, + "step": 150 + }, + { + "clip_ratio": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 346.025, + "completions/max_terminated_length": 346.025, + "completions/mean_length": 229.28125, + "completions/mean_terminated_length": 229.28125, + "completions/min_length": 140.1375, + "completions/min_terminated_length": 140.1375, + "epoch": 0.2733083535375799, + "grad_norm": 0.34731522656971436, + "kl": 0.011865234375, + "learning_rate": 1e-06, + "loss": 0.0106, + "num_tokens": 23184281.0, + "reward": 0.8204727115109562, + "reward_std": 0.12891971635399385, + "rewards/format_reward/mean": 1.0, + "rewards/format_reward/std": 0.0, + "rewards/qatch_metrics/mean": 0.788791407039389, + "rewards/qatch_metrics/std": 0.15167025988921523, + "rewards/tag_count_reward/mean": 1.0, + "rewards/tag_count_reward/std": 0.0, + "step": 155 + }, + { + "clip_ratio": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 318.55, + "completions/max_terminated_length": 318.55, + "completions/mean_length": 209.17734375, + "completions/mean_terminated_length": 209.17734375, + "completions/min_length": 137.3625, + "completions/min_terminated_length": 137.3625, + "epoch": 0.28212475203879217, + "grad_norm": 0.3719362212745786, + "kl": 0.011676025390625, + "learning_rate": 1e-06, + "loss": 0.0151, + "num_tokens": 23893036.0, + "reward": 0.7991067057475447, + "reward_std": 0.09515065372979734, + "rewards/format_reward/mean": 0.99921875, + "rewards/format_reward/std": 0.003125, + "rewards/qatch_metrics/mean": 0.763758339243941, + "rewards/qatch_metrics/std": 0.1118375029604067, + "rewards/tag_count_reward/mean": 0.9998046875, + "rewards/tag_count_reward/std": 0.00078125, + "step": 160 + }, + { + "clip_ratio": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 303.1375, + "completions/max_terminated_length": 303.1375, + "completions/mean_length": 201.36484375, + "completions/mean_terminated_length": 201.36484375, + "completions/min_length": 133.0625, + "completions/min_terminated_length": 133.0625, + "epoch": 0.2909411505400044, + "grad_norm": 0.41140648881811986, + "kl": 0.014212799072265626, + "learning_rate": 1e-06, + "loss": -0.0031, + "num_tokens": 24627135.0, + "reward": 0.7533468393608928, + "reward_std": 0.10053728537168354, + "rewards/format_reward/mean": 1.0, + "rewards/format_reward/std": 0.0, + "rewards/qatch_metrics/mean": 0.7098197954706847, + "rewards/qatch_metrics/std": 0.11827916505280882, + "rewards/tag_count_reward/mean": 1.0, + "rewards/tag_count_reward/std": 0.0, + "step": 165 + }, + { + "clip_ratio": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 308.0, + "completions/max_terminated_length": 308.0, + "completions/mean_length": 203.5234375, + "completions/mean_terminated_length": 203.5234375, + "completions/min_length": 131.275, + "completions/min_terminated_length": 131.275, + "epoch": 0.29975754904121665, + "grad_norm": 0.40595987740034806, + "kl": 0.01388092041015625, + "learning_rate": 1e-06, + "loss": 0.0009, + "num_tokens": 25368413.0, + "reward": 0.7701841354370117, + "reward_std": 0.09663771950872616, + "rewards/format_reward/mean": 1.0, + "rewards/format_reward/std": 0.0, + "rewards/qatch_metrics/mean": 0.7296283877098176, + "rewards/qatch_metrics/std": 0.11369143746051122, + "rewards/tag_count_reward/mean": 1.0, + "rewards/tag_count_reward/std": 0.0, + "step": 170 + }, + { + "clip_ratio": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 328.725, + "completions/max_terminated_length": 328.725, + "completions/mean_length": 216.08125, + "completions/mean_terminated_length": 216.08125, + "completions/min_length": 138.325, + "completions/min_terminated_length": 138.325, + "epoch": 0.3085739475424289, + "grad_norm": 0.25083299753484695, + "kl": 0.01239776611328125, + "learning_rate": 1e-06, + "loss": 0.0008, + "num_tokens": 26110613.0, + "reward": 0.8106268728151917, + "reward_std": 0.10756922718137503, + "rewards/format_reward/mean": 1.0, + "rewards/format_reward/std": 0.0, + "rewards/qatch_metrics/mean": 0.7772080772556365, + "rewards/qatch_metrics/std": 0.12655203738249837, + "rewards/tag_count_reward/mean": 1.0, + "rewards/tag_count_reward/std": 0.0, + "step": 175 + }, + { + "clip_ratio": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 343.4625, + "completions/max_terminated_length": 343.4625, + "completions/mean_length": 231.24453125, + "completions/mean_terminated_length": 231.24453125, + "completions/min_length": 149.375, + "completions/min_terminated_length": 149.375, + "epoch": 0.3173903460436412, + "grad_norm": 0.3299647961730041, + "kl": 0.012017822265625, + "learning_rate": 1e-06, + "loss": -0.0011, + "num_tokens": 26876590.0, + "reward": 0.7713820965960622, + "reward_std": 0.11187393190339208, + "rewards/format_reward/mean": 0.99921875, + "rewards/format_reward/std": 0.003125, + "rewards/qatch_metrics/mean": 0.7311411482747644, + "rewards/qatch_metrics/std": 0.13165212250314653, + "rewards/tag_count_reward/mean": 0.9998046875, + "rewards/tag_count_reward/std": 0.00078125, + "step": 180 + }, + { + "clip_ratio": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 353.5875, + "completions/max_terminated_length": 353.5875, + "completions/mean_length": 242.12734375, + "completions/mean_terminated_length": 242.12734375, + "completions/min_length": 155.3125, + "completions/min_terminated_length": 155.3125, + "epoch": 0.32620674454485343, + "grad_norm": 0.32558550372348305, + "kl": 0.013112258911132813, + "learning_rate": 1e-06, + "loss": 0.0031, + "num_tokens": 27658417.0, + "reward": 0.8029140060767531, + "reward_std": 0.10181500271428376, + "rewards/format_reward/mean": 1.0, + "rewards/format_reward/std": 0.0, + "rewards/qatch_metrics/mean": 0.7681341207586229, + "rewards/qatch_metrics/std": 0.1197823622263968, + "rewards/tag_count_reward/mean": 1.0, + "rewards/tag_count_reward/std": 0.0, + "step": 185 + }, + { + "clip_ratio": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 358.3625, + "completions/max_terminated_length": 358.3625, + "completions/mean_length": 244.378125, + "completions/mean_terminated_length": 244.378125, + "completions/min_length": 150.175, + "completions/min_terminated_length": 150.175, + "epoch": 0.3350231430460657, + "grad_norm": 0.32369260650095066, + "kl": 0.012503814697265626, + "learning_rate": 1e-06, + "loss": 0.0035, + "num_tokens": 28450773.0, + "reward": 0.8509975776076317, + "reward_std": 0.10326635130477371, + "rewards/format_reward/mean": 0.99921875, + "rewards/format_reward/std": 0.003125, + "rewards/qatch_metrics/mean": 0.8248408894985915, + "rewards/qatch_metrics/std": 0.12098856130687637, + "rewards/tag_count_reward/mean": 0.99921875, + "rewards/tag_count_reward/std": 0.003125, + "step": 190 + }, + { + "clip_ratio": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 346.8, + "completions/max_terminated_length": 346.8, + "completions/mean_length": 229.31484375, + "completions/mean_terminated_length": 229.31484375, + "completions/min_length": 142.2, + "completions/min_terminated_length": 142.2, + "epoch": 0.3438395415472779, + "grad_norm": 0.33109392130180526, + "kl": 0.012652969360351563, + "learning_rate": 1e-06, + "loss": 0.005, + "num_tokens": 29193720.0, + "reward": 0.8215816805139184, + "reward_std": 0.10861785978777334, + "rewards/format_reward/mean": 0.99921875, + "rewards/format_reward/std": 0.003125, + "rewards/qatch_metrics/mean": 0.7901994839310647, + "rewards/qatch_metrics/std": 0.12737212259089575, + "rewards/tag_count_reward/mean": 0.9998046875, + "rewards/tag_count_reward/std": 0.00078125, + "step": 195 + }, + { + "clip_ratio": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 332.9, + "completions/max_terminated_length": 332.9, + "completions/mean_length": 226.34296875, + "completions/mean_terminated_length": 226.34296875, + "completions/min_length": 144.8, + "completions/min_terminated_length": 144.8, + "epoch": 0.3526559400484902, + "grad_norm": 0.34898255755360424, + "kl": 0.0120391845703125, + "learning_rate": 1e-06, + "loss": -0.001, + "num_tokens": 29936351.0, + "reward": 0.8118854926899075, + "reward_std": 0.08703446270665154, + "rewards/format_reward/mean": 1.0, + "rewards/format_reward/std": 0.0, + "rewards/qatch_metrics/mean": 0.778688807785511, + "rewards/qatch_metrics/std": 0.10239349115872756, + "rewards/tag_count_reward/mean": 1.0, + "rewards/tag_count_reward/std": 0.0, + "step": 200 + }, + { + "clip_ratio": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 341.4875, + "completions/max_terminated_length": 341.4875, + "completions/mean_length": 233.30625, + "completions/mean_terminated_length": 233.30625, + "completions/min_length": 146.125, + "completions/min_terminated_length": 146.125, + "epoch": 0.36147233854970245, + "grad_norm": 0.2847757548262052, + "kl": 0.013054656982421874, + "learning_rate": 1e-06, + "loss": 0.0005, + "num_tokens": 30723207.0, + "reward": 0.8451463960111141, + "reward_std": 0.07684196562040597, + "rewards/format_reward/mean": 1.0, + "rewards/format_reward/std": 0.0, + "rewards/qatch_metrics/mean": 0.8178192753344774, + "rewards/qatch_metrics/std": 0.09040231805993244, + "rewards/tag_count_reward/mean": 1.0, + "rewards/tag_count_reward/std": 0.0, + "step": 205 + }, + { + "clip_ratio": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 344.9125, + "completions/max_terminated_length": 344.9125, + "completions/mean_length": 233.31953125, + "completions/mean_terminated_length": 233.31953125, + "completions/min_length": 144.9875, + "completions/min_terminated_length": 144.9875, + "epoch": 0.3702887370509147, + "grad_norm": 0.4034177578209816, + "kl": 0.014620590209960937, + "learning_rate": 1e-06, + "loss": 0.0075, + "num_tokens": 31508960.0, + "reward": 0.7541328657418489, + "reward_std": 0.10346053875982761, + "rewards/format_reward/mean": 1.0, + "rewards/format_reward/std": 0.0, + "rewards/qatch_metrics/mean": 0.7107445362955331, + "rewards/qatch_metrics/std": 0.12171828672289849, + "rewards/tag_count_reward/mean": 1.0, + "rewards/tag_count_reward/std": 0.0, + "step": 210 + }, + { + "clip_ratio": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 337.2125, + "completions/max_terminated_length": 337.2125, + "completions/mean_length": 229.95234375, + "completions/mean_terminated_length": 229.95234375, + "completions/min_length": 143.85, + "completions/min_terminated_length": 143.85, + "epoch": 0.37910513555212694, + "grad_norm": 0.3513936634634679, + "kl": 0.01492767333984375, + "learning_rate": 1e-06, + "loss": -0.0053, + "num_tokens": 32267923.0, + "reward": 0.7756918715313077, + "reward_std": 0.08237711414694786, + "rewards/format_reward/mean": 1.0, + "rewards/format_reward/std": 0.0, + "rewards/qatch_metrics/mean": 0.7361080780159682, + "rewards/qatch_metrics/std": 0.09691425783094018, + "rewards/tag_count_reward/mean": 1.0, + "rewards/tag_count_reward/std": 0.0, + "step": 215 + }, + { + "clip_ratio": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 331.65, + "completions/max_terminated_length": 331.65, + "completions/mean_length": 225.4984375, + "completions/mean_terminated_length": 225.4984375, + "completions/min_length": 145.2, + "completions/min_terminated_length": 145.2, + "epoch": 0.38792153405333923, + "grad_norm": 0.2816074681768635, + "kl": 0.0148773193359375, + "learning_rate": 1e-06, + "loss": -0.0012, + "num_tokens": 33038177.0, + "reward": 0.8032938608899712, + "reward_std": 0.09598501106374897, + "rewards/format_reward/mean": 1.0, + "rewards/format_reward/std": 0.0, + "rewards/qatch_metrics/mean": 0.7685809925198555, + "rewards/qatch_metrics/std": 0.11292354888282716, + "rewards/tag_count_reward/mean": 1.0, + "rewards/tag_count_reward/std": 0.0, + "step": 220 + }, + { + "clip_ratio": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 316.7, + "completions/max_terminated_length": 316.7, + "completions/mean_length": 215.88984375, + "completions/mean_terminated_length": 215.88984375, + "completions/min_length": 138.5125, + "completions/min_terminated_length": 138.5125, + "epoch": 0.3967379325545515, + "grad_norm": 0.2792708523998877, + "kl": 0.01438140869140625, + "learning_rate": 1e-06, + "loss": -0.0031, + "num_tokens": 33775780.0, + "reward": 0.833816378749907, + "reward_std": 0.08319322268362157, + "rewards/format_reward/mean": 1.0, + "rewards/format_reward/std": 0.0, + "rewards/qatch_metrics/mean": 0.8044898479929543, + "rewards/qatch_metrics/std": 0.09787438737985213, + "rewards/tag_count_reward/mean": 1.0, + "rewards/tag_count_reward/std": 0.0, + "step": 225 + }, + { + "clip_ratio": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 321.1625, + "completions/max_terminated_length": 321.1625, + "completions/mean_length": 212.334375, + "completions/mean_terminated_length": 212.334375, + "completions/min_length": 140.6, + "completions/min_terminated_length": 140.6, + "epoch": 0.4055543310557637, + "grad_norm": 0.2700162867921189, + "kl": 0.015807723999023436, + "learning_rate": 1e-06, + "loss": -0.0029, + "num_tokens": 34518032.0, + "reward": 0.7357370050624013, + "reward_std": 0.10150592336431145, + "rewards/format_reward/mean": 1.0, + "rewards/format_reward/std": 0.0, + "rewards/qatch_metrics/mean": 0.6891023456788389, + "rewards/qatch_metrics/std": 0.11941873789764941, + "rewards/tag_count_reward/mean": 1.0, + "rewards/tag_count_reward/std": 0.0, + "step": 230 + }, + { + "clip_ratio": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 325.4625, + "completions/max_terminated_length": 325.4625, + "completions/mean_length": 218.98125, + "completions/mean_terminated_length": 218.98125, + "completions/min_length": 143.225, + "completions/min_terminated_length": 143.225, + "epoch": 0.41437072955697596, + "grad_norm": 0.41442176380867557, + "kl": 0.014601898193359376, + "learning_rate": 1e-06, + "loss": 0.0086, + "num_tokens": 35259240.0, + "reward": 0.7820553651079536, + "reward_std": 0.112267074175179, + "rewards/format_reward/mean": 1.0, + "rewards/format_reward/std": 0.0, + "rewards/qatch_metrics/mean": 0.743594534881413, + "rewards/qatch_metrics/std": 0.13207891816273332, + "rewards/tag_count_reward/mean": 1.0, + "rewards/tag_count_reward/std": 0.0, + "step": 235 + }, + { + "clip_ratio": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 319.25, + "completions/max_terminated_length": 319.25, + "completions/mean_length": 209.9296875, + "completions/mean_terminated_length": 209.9296875, + "completions/min_length": 137.3125, + "completions/min_terminated_length": 137.3125, + "epoch": 0.42318712805818826, + "grad_norm": 0.3476187294524526, + "kl": 0.014452362060546875, + "learning_rate": 1e-06, + "loss": -0.002, + "num_tokens": 35975326.0, + "reward": 0.7921670457348228, + "reward_std": 0.09589202178794949, + "rewards/format_reward/mean": 1.0, + "rewards/format_reward/std": 0.0, + "rewards/qatch_metrics/mean": 0.755490631237626, + "rewards/qatch_metrics/std": 0.1128141468463582, + "rewards/tag_count_reward/mean": 1.0, + "rewards/tag_count_reward/std": 0.0, + "step": 240 + }, + { + "clip_ratio": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 330.4, + "completions/max_terminated_length": 330.4, + "completions/mean_length": 219.6890625, + "completions/mean_terminated_length": 219.6890625, + "completions/min_length": 139.175, + "completions/min_terminated_length": 139.175, + "epoch": 0.4320035265594005, + "grad_norm": 0.24104253998662034, + "kl": 0.014841079711914062, + "learning_rate": 1e-06, + "loss": 0.0048, + "num_tokens": 36727824.0, + "reward": 0.799939620308578, + "reward_std": 0.10737682661347207, + "rewards/format_reward/mean": 1.0, + "rewards/format_reward/std": 0.0, + "rewards/qatch_metrics/mean": 0.7646578153668088, + "rewards/qatch_metrics/std": 0.1263537659888243, + "rewards/tag_count_reward/mean": 0.999609375, + "rewards/tag_count_reward/std": 0.0015625, + "step": 245 + }, + { + "clip_ratio": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 327.9375, + "completions/max_terminated_length": 327.9375, + "completions/mean_length": 218.1921875, + "completions/mean_terminated_length": 218.1921875, + "completions/min_length": 140.15, + "completions/min_terminated_length": 140.15, + "epoch": 0.44081992506061274, + "grad_norm": 0.2539246494683978, + "kl": 0.014622879028320313, + "learning_rate": 1e-06, + "loss": -0.0025, + "num_tokens": 37470134.0, + "reward": 0.7929010545834899, + "reward_std": 0.08832431975752116, + "rewards/format_reward/mean": 1.0, + "rewards/format_reward/std": 0.0, + "rewards/qatch_metrics/mean": 0.7563541710143908, + "rewards/qatch_metrics/std": 0.10391096852254122, + "rewards/tag_count_reward/mean": 1.0, + "rewards/tag_count_reward/std": 0.0, + "step": 250 + }, + { + "clip_ratio": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 369.9625, + "completions/max_terminated_length": 369.9625, + "completions/mean_length": 228.9, + "completions/mean_terminated_length": 228.9, + "completions/min_length": 142.2125, + "completions/min_terminated_length": 142.2125, + "epoch": 0.449636323561825, + "grad_norm": 0.389718556100374, + "kl": 0.013026809692382813, + "learning_rate": 1e-06, + "loss": 0.0146, + "num_tokens": 38224342.0, + "reward": 0.8210916120558978, + "reward_std": 0.08054333752952517, + "rewards/format_reward/mean": 1.0, + "rewards/format_reward/std": 0.0, + "rewards/qatch_metrics/mean": 0.7895195348886773, + "rewards/qatch_metrics/std": 0.0947568719740957, + "rewards/tag_count_reward/mean": 1.0, + "rewards/tag_count_reward/std": 0.0, + "step": 255 + }, + { + "clip_ratio": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 360.125, + "completions/max_terminated_length": 360.125, + "completions/mean_length": 236.08203125, + "completions/mean_terminated_length": 236.08203125, + "completions/min_length": 150.35, + "completions/min_terminated_length": 150.35, + "epoch": 0.4584527220630373, + "grad_norm": 0.31386840054419024, + "kl": 0.01431121826171875, + "learning_rate": 1e-06, + "loss": -0.0003, + "num_tokens": 39005375.0, + "reward": 0.7916189678013325, + "reward_std": 0.10870604729279876, + "rewards/format_reward/mean": 1.0, + "rewards/format_reward/std": 0.0, + "rewards/qatch_metrics/mean": 0.7548458357341588, + "rewards/qatch_metrics/std": 0.12788947536610068, + "rewards/tag_count_reward/mean": 1.0, + "rewards/tag_count_reward/std": 0.0, + "step": 260 + }, + { + "clip_ratio": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 361.3, + "completions/max_terminated_length": 361.3, + "completions/mean_length": 241.08046875, + "completions/mean_terminated_length": 241.08046875, + "completions/min_length": 143.625, + "completions/min_terminated_length": 143.625, + "epoch": 0.4672691205642495, + "grad_norm": 0.32199129625786044, + "kl": 0.014544677734375, + "learning_rate": 1e-06, + "loss": 0.0076, + "num_tokens": 39806326.0, + "reward": 0.8503130197525024, + "reward_std": 0.08854990721156356, + "rewards/format_reward/mean": 1.0, + "rewards/format_reward/std": 0.0, + "rewards/qatch_metrics/mean": 0.8238976599182933, + "rewards/qatch_metrics/std": 0.10417636758938897, + "rewards/tag_count_reward/mean": 1.0, + "rewards/tag_count_reward/std": 0.0, + "step": 265 + }, + { + "clip_ratio": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 357.9125, + "completions/max_terminated_length": 357.9125, + "completions/mean_length": 238.00703125, + "completions/mean_terminated_length": 238.00703125, + "completions/min_length": 150.7125, + "completions/min_terminated_length": 150.7125, + "epoch": 0.47608551906546176, + "grad_norm": 0.24307758730384177, + "kl": 0.0154083251953125, + "learning_rate": 1e-06, + "loss": 0.0007, + "num_tokens": 40579551.0, + "reward": 0.7649708043783903, + "reward_std": 0.09011161667294801, + "rewards/format_reward/mean": 1.0, + "rewards/format_reward/std": 0.0, + "rewards/qatch_metrics/mean": 0.7234950542595471, + "rewards/qatch_metrics/std": 0.1060136711690575, + "rewards/tag_count_reward/mean": 1.0, + "rewards/tag_count_reward/std": 0.0, + "step": 270 + }, + { + "clip_ratio": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 329.375, + "completions/max_terminated_length": 329.375, + "completions/mean_length": 218.865625, + "completions/mean_terminated_length": 218.865625, + "completions/min_length": 136.9375, + "completions/min_terminated_length": 136.9375, + "epoch": 0.484901917566674, + "grad_norm": 0.2437912279556105, + "kl": 0.01680908203125, + "learning_rate": 1e-06, + "loss": 0.0116, + "num_tokens": 41304819.0, + "reward": 0.7731978652998805, + "reward_std": 0.09575790755916387, + "rewards/format_reward/mean": 0.99921875, + "rewards/format_reward/std": 0.003125, + "rewards/qatch_metrics/mean": 0.7332773502916098, + "rewards/qatch_metrics/std": 0.11264292849227786, + "rewards/tag_count_reward/mean": 0.9998046875, + "rewards/tag_count_reward/std": 0.00078125, + "step": 275 + }, + { + "clip_ratio": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 297.425, + "completions/max_terminated_length": 297.425, + "completions/mean_length": 194.35234375, + "completions/mean_terminated_length": 194.35234375, + "completions/min_length": 129.725, + "completions/min_terminated_length": 129.725, + "epoch": 0.49371831606788624, + "grad_norm": 0.3022726877012163, + "kl": 0.017235565185546874, + "learning_rate": 1e-06, + "loss": 0.0071, + "num_tokens": 41986038.0, + "reward": 0.8300179397687316, + "reward_std": 0.08664830076304497, + "rewards/format_reward/mean": 1.0, + "rewards/format_reward/std": 0.0, + "rewards/qatch_metrics/mean": 0.8000210979953408, + "rewards/qatch_metrics/std": 0.10193918244767701, + "rewards/tag_count_reward/mean": 1.0, + "rewards/tag_count_reward/std": 0.0, + "step": 280 + }, + { + "clip_ratio": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 276.7875, + "completions/max_terminated_length": 276.7875, + "completions/mean_length": 183.3453125, + "completions/mean_terminated_length": 183.3453125, + "completions/min_length": 127.55, + "completions/min_terminated_length": 127.55, + "epoch": 0.5025347145690985, + "grad_norm": 0.22342235477555344, + "kl": 0.018723297119140624, + "learning_rate": 1e-06, + "loss": -0.0009, + "num_tokens": 42654592.0, + "reward": 0.8566583547741174, + "reward_std": 0.08069641448455514, + "rewards/format_reward/mean": 1.0, + "rewards/format_reward/std": 0.0, + "rewards/qatch_metrics/mean": 0.8313627634197474, + "rewards/qatch_metrics/std": 0.09493696391291451, + "rewards/tag_count_reward/mean": 1.0, + "rewards/tag_count_reward/std": 0.0, + "step": 285 + }, + { + "clip_ratio": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 331.125, + "completions/max_terminated_length": 331.125, + "completions/mean_length": 220.7734375, + "completions/mean_terminated_length": 220.7734375, + "completions/min_length": 140.35, + "completions/min_terminated_length": 140.35, + "epoch": 0.5113511130703108, + "grad_norm": 0.34607047517160744, + "kl": 0.01548309326171875, + "learning_rate": 1e-06, + "loss": -0.006, + "num_tokens": 43404558.0, + "reward": 0.8139310251921416, + "reward_std": 0.0980742353014648, + "rewards/format_reward/mean": 1.0, + "rewards/format_reward/std": 0.0, + "rewards/qatch_metrics/mean": 0.7810953183798119, + "rewards/qatch_metrics/std": 0.11538145933300256, + "rewards/tag_count_reward/mean": 1.0, + "rewards/tag_count_reward/std": 0.0, + "step": 290 + }, + { + "clip_ratio": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 372.3625, + "completions/max_terminated_length": 372.3625, + "completions/mean_length": 253.30859375, + "completions/mean_terminated_length": 253.30859375, + "completions/min_length": 162.3375, + "completions/min_terminated_length": 162.3375, + "epoch": 0.5201675115715231, + "grad_norm": 0.26233887817165846, + "kl": 0.017038726806640626, + "learning_rate": 1e-06, + "loss": -0.0014, + "num_tokens": 44202393.0, + "reward": 0.7521807491779328, + "reward_std": 0.0808277386619011, + "rewards/format_reward/mean": 1.0, + "rewards/format_reward/std": 0.0, + "rewards/qatch_metrics/mean": 0.7084479197394102, + "rewards/qatch_metrics/std": 0.09509145880147116, + "rewards/tag_count_reward/mean": 1.0, + "rewards/tag_count_reward/std": 0.0, + "step": 295 + }, + { + "clip_ratio": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 371.2875, + "completions/max_terminated_length": 371.2875, + "completions/mean_length": 256.74375, + "completions/mean_terminated_length": 256.74375, + "completions/min_length": 164.575, + "completions/min_terminated_length": 164.575, + "epoch": 0.5289839100727353, + "grad_norm": 0.27528750385577455, + "kl": 0.0166046142578125, + "learning_rate": 1e-06, + "loss": 0.0034, + "num_tokens": 44996449.0, + "reward": 0.7653900509700179, + "reward_std": 0.10405777737032622, + "rewards/format_reward/mean": 1.0, + "rewards/format_reward/std": 0.0, + "rewards/qatch_metrics/mean": 0.7239882865447725, + "rewards/qatch_metrics/std": 0.12242091761436313, + "rewards/tag_count_reward/mean": 1.0, + "rewards/tag_count_reward/std": 0.0, + "step": 300 + }, + { + "clip_ratio": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 383.75, + "completions/max_terminated_length": 383.75, + "completions/mean_length": 262.81875, + "completions/mean_terminated_length": 262.81875, + "completions/min_length": 163.2, + "completions/min_terminated_length": 163.2, + "epoch": 0.5378003085739476, + "grad_norm": 0.2916124436851988, + "kl": 0.016686248779296874, + "learning_rate": 1e-06, + "loss": -0.0091, + "num_tokens": 45810553.0, + "reward": 0.751503630913794, + "reward_std": 0.10126840746961534, + "rewards/format_reward/mean": 1.0, + "rewards/format_reward/std": 0.0, + "rewards/qatch_metrics/mean": 0.7076513038016856, + "rewards/qatch_metrics/std": 0.1191393076442182, + "rewards/tag_count_reward/mean": 1.0, + "rewards/tag_count_reward/std": 0.0, + "step": 305 + }, + { + "clip_ratio": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 385.4, + "completions/max_terminated_length": 385.4, + "completions/mean_length": 255.634375, + "completions/mean_terminated_length": 255.634375, + "completions/min_length": 157.3125, + "completions/min_terminated_length": 157.3125, + "epoch": 0.5466167070751597, + "grad_norm": 0.3015279724579697, + "kl": 0.017456817626953124, + "learning_rate": 1e-06, + "loss": 0.0074, + "num_tokens": 46626773.0, + "reward": 0.7629254685714841, + "reward_std": 0.14258774023037404, + "rewards/format_reward/mean": 1.0, + "rewards/format_reward/std": 0.0, + "rewards/qatch_metrics/mean": 0.721100265783025, + "rewards/qatch_metrics/std": 0.16777858033310622, + "rewards/tag_count_reward/mean": 0.9998046875, + "rewards/tag_count_reward/std": 0.00078125, + "step": 310 + }, + { + "clip_ratio": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 331.5375, + "completions/max_terminated_length": 331.5375, + "completions/mean_length": 217.5625, + "completions/mean_terminated_length": 217.5625, + "completions/min_length": 134.5375, + "completions/min_terminated_length": 134.5375, + "epoch": 0.555433105576372, + "grad_norm": 0.2736624916091261, + "kl": 0.016427993774414062, + "learning_rate": 1e-06, + "loss": 0.0011, + "num_tokens": 47337797.0, + "reward": 0.8417512066662312, + "reward_std": 0.06132402071962133, + "rewards/format_reward/mean": 1.0, + "rewards/format_reward/std": 0.0, + "rewards/qatch_metrics/mean": 0.8138479188084602, + "rewards/qatch_metrics/std": 0.07210502420784906, + "rewards/tag_count_reward/mean": 0.999609375, + "rewards/tag_count_reward/std": 0.0015625, + "step": 315 + }, + { + "clip_ratio": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 347.9625, + "completions/max_terminated_length": 347.9625, + "completions/mean_length": 225.61484375, + "completions/mean_terminated_length": 225.61484375, + "completions/min_length": 139.5625, + "completions/min_terminated_length": 139.5625, + "epoch": 0.5642495040775843, + "grad_norm": 0.3231991864642648, + "kl": 0.017105865478515624, + "learning_rate": 1e-06, + "loss": 0.0004, + "num_tokens": 48108472.0, + "reward": 0.8285636451095343, + "reward_std": 0.09632078433714923, + "rewards/format_reward/mean": 1.0, + "rewards/format_reward/std": 0.0, + "rewards/qatch_metrics/mean": 0.7983101591467857, + "rewards/qatch_metrics/std": 0.11331857727491297, + "rewards/tag_count_reward/mean": 1.0, + "rewards/tag_count_reward/std": 0.0, + "step": 320 + }, + { + "clip_ratio": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 347.45, + "completions/max_terminated_length": 347.45, + "completions/mean_length": 231.76328125, + "completions/mean_terminated_length": 231.76328125, + "completions/min_length": 148.275, + "completions/min_terminated_length": 148.275, + "epoch": 0.5730659025787965, + "grad_norm": 0.2973174138867235, + "kl": 0.016844940185546876, + "learning_rate": 1e-06, + "loss": 0.0095, + "num_tokens": 48899481.0, + "reward": 0.7851231107488275, + "reward_std": 0.09552590373905331, + "rewards/format_reward/mean": 1.0, + "rewards/format_reward/std": 0.0, + "rewards/qatch_metrics/mean": 0.7472036501319508, + "rewards/qatch_metrics/std": 0.11238342153937993, + "rewards/tag_count_reward/mean": 1.0, + "rewards/tag_count_reward/std": 0.0, + "step": 325 + }, + { + "clip_ratio": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 359.275, + "completions/max_terminated_length": 359.275, + "completions/mean_length": 234.6609375, + "completions/mean_terminated_length": 234.6609375, + "completions/min_length": 146.075, + "completions/min_terminated_length": 146.075, + "epoch": 0.5818823010800088, + "grad_norm": 0.3288208170160159, + "kl": 0.017102813720703124, + "learning_rate": 1e-06, + "loss": 0.0158, + "num_tokens": 49645255.0, + "reward": 0.8200173564255238, + "reward_std": 0.09870836285626865, + "rewards/format_reward/mean": 1.0, + "rewards/format_reward/std": 0.0, + "rewards/qatch_metrics/mean": 0.7882671934901737, + "rewards/qatch_metrics/std": 0.11608153889974346, + "rewards/tag_count_reward/mean": 0.9998046875, + "rewards/tag_count_reward/std": 0.00078125, + "step": 330 + }, + { + "clip_ratio": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 357.4375, + "completions/max_terminated_length": 357.4375, + "completions/mean_length": 230.21484375, + "completions/mean_terminated_length": 230.21484375, + "completions/min_length": 140.6625, + "completions/min_terminated_length": 140.6625, + "epoch": 0.5906986995812211, + "grad_norm": 0.4614331608564361, + "kl": 0.018951416015625, + "learning_rate": 1e-06, + "loss": 0.0084, + "num_tokens": 50437466.0, + "reward": 0.7783582922071218, + "reward_std": 0.09422233710065483, + "rewards/format_reward/mean": 0.99921875, + "rewards/format_reward/std": 0.003125, + "rewards/qatch_metrics/mean": 0.7393484384519979, + "rewards/qatch_metrics/std": 0.1104362107347697, + "rewards/tag_count_reward/mean": 0.9998046875, + "rewards/tag_count_reward/std": 0.00078125, + "step": 335 + }, + { + "clip_ratio": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 324.8875, + "completions/max_terminated_length": 324.8875, + "completions/mean_length": 209.18125, + "completions/mean_terminated_length": 209.18125, + "completions/min_length": 134.8375, + "completions/min_terminated_length": 134.8375, + "epoch": 0.5995150980824333, + "grad_norm": 0.2725327541303037, + "kl": 0.019439697265625, + "learning_rate": 1e-06, + "loss": 0.0057, + "num_tokens": 51167586.0, + "reward": 0.8479114411398768, + "reward_std": 0.07958720491733402, + "rewards/format_reward/mean": 0.9984375, + "rewards/format_reward/std": 0.00625, + "rewards/qatch_metrics/mean": 0.8213135461322963, + "rewards/qatch_metrics/std": 0.0926669392734766, + "rewards/tag_count_reward/mean": 0.9990234375, + "rewards/tag_count_reward/std": 0.00390625, + "step": 340 + }, + { + "clip_ratio": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 345.0833333333333, + "completions/max_terminated_length": 345.0833333333333, + "completions/mean_length": 229.90755208333334, + "completions/mean_terminated_length": 229.90755208333334, + "completions/min_length": 146.02083333333334, + "completions/min_terminated_length": 146.02083333333334, + "epoch": 0.6083314965836456, + "grad_norm": 0.33113384827355125, + "kl": 0.019663492838541668, + "learning_rate": 1e-06, + "loss": 0.0054, + "num_tokens": 51884877.0, + "reward": 0.7889465639988581, + "reward_std": 0.09934755021822639, + "rewards/format_reward/mean": 1.0, + "rewards/format_reward/std": 0.0, + "rewards/qatch_metrics/mean": 0.7517018277818958, + "rewards/qatch_metrics/std": 0.11687947452689211, + "rewards/tag_count_reward/mean": 1.0, + "rewards/tag_count_reward/std": 0.0, + "step": 345 + }, + { + "clip_ratio": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 336.4125, + "completions/max_terminated_length": 336.4125, + "completions/mean_length": 225.73203125, + "completions/mean_terminated_length": 225.73203125, + "completions/min_length": 142.075, + "completions/min_terminated_length": 142.075, + "epoch": 0.6171478950848578, + "grad_norm": 0.3566644104382955, + "kl": 0.020180511474609374, + "learning_rate": 1e-06, + "loss": -0.0094, + "num_tokens": 52662630.0, + "reward": 0.8238930691033601, + "reward_std": 0.08169953491305933, + "rewards/format_reward/mean": 1.0, + "rewards/format_reward/std": 0.0, + "rewards/qatch_metrics/mean": 0.7928153664804996, + "rewards/qatch_metrics/std": 0.09611710296012461, + "rewards/tag_count_reward/mean": 1.0, + "rewards/tag_count_reward/std": 0.0, + "step": 350 + }, + { + "clip_ratio": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 343.2, + "completions/max_terminated_length": 343.2, + "completions/mean_length": 236.98046875, + "completions/mean_terminated_length": 236.98046875, + "completions/min_length": 150.9, + "completions/min_terminated_length": 150.9, + "epoch": 0.6259642935860701, + "grad_norm": 0.28516393114431005, + "kl": 0.017165374755859376, + "learning_rate": 1e-06, + "loss": 0.0005, + "num_tokens": 53419117.0, + "reward": 0.8083362869918347, + "reward_std": 0.0969126635696739, + "rewards/format_reward/mean": 0.99921875, + "rewards/format_reward/std": 0.003125, + "rewards/qatch_metrics/mean": 0.7746166706085205, + "rewards/qatch_metrics/std": 0.11360130067914724, + "rewards/tag_count_reward/mean": 0.9998046875, + "rewards/tag_count_reward/std": 0.00078125, + "step": 355 + }, + { + "clip_ratio": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 352.6875, + "completions/max_terminated_length": 352.6875, + "completions/mean_length": 239.6953125, + "completions/mean_terminated_length": 239.6953125, + "completions/min_length": 146.55, + "completions/min_terminated_length": 146.55, + "epoch": 0.6347806920872824, + "grad_norm": 0.17994755171571641, + "kl": 0.017398834228515625, + "learning_rate": 1e-06, + "loss": 0.0008, + "num_tokens": 54185031.0, + "reward": 0.8027490990236401, + "reward_std": 0.05896135854927707, + "rewards/format_reward/mean": 1.0, + "rewards/format_reward/std": 0.0, + "rewards/qatch_metrics/mean": 0.7679401046669228, + "rewards/qatch_metrics/std": 0.06936630747477465, + "rewards/tag_count_reward/mean": 1.0, + "rewards/tag_count_reward/std": 0.0, + "step": 360 + }, + { + "clip_ratio": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 352.7125, + "completions/max_terminated_length": 352.7125, + "completions/mean_length": 239.52109375, + "completions/mean_terminated_length": 239.52109375, + "completions/min_length": 145.0125, + "completions/min_terminated_length": 145.0125, + "epoch": 0.6435970905884946, + "grad_norm": 0.23679998463114285, + "kl": 0.017606353759765624, + "learning_rate": 1e-06, + "loss": -0.0013, + "num_tokens": 54936114.0, + "reward": 0.8317079545930028, + "reward_std": 0.08216131356894038, + "rewards/format_reward/mean": 1.0, + "rewards/format_reward/std": 0.0, + "rewards/qatch_metrics/mean": 0.8020208382047713, + "rewards/qatch_metrics/std": 0.0966144205071032, + "rewards/tag_count_reward/mean": 0.9998046875, + "rewards/tag_count_reward/std": 0.00078125, + "step": 365 + }, + { + "clip_ratio": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 368.05, + "completions/max_terminated_length": 368.05, + "completions/mean_length": 253.25546875, + "completions/mean_terminated_length": 253.25546875, + "completions/min_length": 161.875, + "completions/min_terminated_length": 161.875, + "epoch": 0.6524134890897069, + "grad_norm": 0.2556963595685909, + "kl": 0.018042755126953126, + "learning_rate": 1e-06, + "loss": 0.0036, + "num_tokens": 55728841.0, + "reward": 0.8250438664108515, + "reward_std": 0.08220216338286264, + "rewards/format_reward/mean": 1.0, + "rewards/format_reward/std": 0.0, + "rewards/qatch_metrics/mean": 0.7941807355555284, + "rewards/qatch_metrics/std": 0.09668861866011866, + "rewards/tag_count_reward/mean": 0.9998046875, + "rewards/tag_count_reward/std": 0.00078125, + "step": 370 + }, + { + "clip_ratio": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 341.6625, + "completions/max_terminated_length": 341.6625, + "completions/mean_length": 237.73046875, + "completions/mean_terminated_length": 237.73046875, + "completions/min_length": 150.525, + "completions/min_terminated_length": 150.525, + "epoch": 0.6612298875909192, + "grad_norm": 0.2751968612090182, + "kl": 0.018801116943359376, + "learning_rate": 1e-06, + "loss": -0.0012, + "num_tokens": 56499232.0, + "reward": 0.8543739832937718, + "reward_std": 0.09150902703240718, + "rewards/format_reward/mean": 1.0, + "rewards/format_reward/std": 0.0, + "rewards/qatch_metrics/mean": 0.828675264492631, + "rewards/qatch_metrics/std": 0.10765768758892591, + "rewards/tag_count_reward/mean": 1.0, + "rewards/tag_count_reward/std": 0.0, + "step": 375 + }, + { + "clip_ratio": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 319.2875, + "completions/max_terminated_length": 319.2875, + "completions/mean_length": 219.77734375, + "completions/mean_terminated_length": 219.77734375, + "completions/min_length": 137.475, + "completions/min_terminated_length": 137.475, + "epoch": 0.6700462860921313, + "grad_norm": 0.31154862475726136, + "kl": 0.01962738037109375, + "learning_rate": 1e-06, + "loss": 0.0002, + "num_tokens": 57237059.0, + "reward": 0.8459062715992332, + "reward_std": 0.08771184119395911, + "rewards/format_reward/mean": 0.9984375, + "rewards/format_reward/std": 0.00625, + "rewards/qatch_metrics/mean": 0.8189200535358395, + "rewards/qatch_metrics/std": 0.1028526050504297, + "rewards/tag_count_reward/mean": 0.999609375, + "rewards/tag_count_reward/std": 0.0015625, + "step": 380 + }, + { + "clip_ratio": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 302.25, + "completions/max_terminated_length": 302.25, + "completions/mean_length": 203.378125, + "completions/mean_terminated_length": 203.378125, + "completions/min_length": 137.75, + "completions/min_terminated_length": 137.75, + "epoch": 0.6788626845933436, + "grad_norm": 0.2352132785923824, + "kl": 0.02192230224609375, + "learning_rate": 1e-06, + "loss": 0.0049, + "num_tokens": 57972807.0, + "reward": 0.82975386492908, + "reward_std": 0.06049516258062795, + "rewards/format_reward/mean": 1.0, + "rewards/format_reward/std": 0.0, + "rewards/qatch_metrics/mean": 0.7997104193782434, + "rewards/qatch_metrics/std": 0.07117078317096456, + "rewards/tag_count_reward/mean": 1.0, + "rewards/tag_count_reward/std": 0.0, + "step": 385 + }, + { + "clip_ratio": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 304.5125, + "completions/max_terminated_length": 304.5125, + "completions/mean_length": 204.80234375, + "completions/mean_terminated_length": 204.80234375, + "completions/min_length": 136.125, + "completions/min_terminated_length": 136.125, + "epoch": 0.6876790830945558, + "grad_norm": 0.27452609767604863, + "kl": 0.022785186767578125, + "learning_rate": 1e-06, + "loss": -0.0009, + "num_tokens": 58735722.0, + "reward": 0.821035155840218, + "reward_std": 0.09460214762948453, + "rewards/format_reward/mean": 0.99921875, + "rewards/format_reward/std": 0.003125, + "rewards/qatch_metrics/mean": 0.7895565141923726, + "rewards/qatch_metrics/std": 0.11137145487591624, + "rewards/tag_count_reward/mean": 0.9998046875, + "rewards/tag_count_reward/std": 0.00078125, + "step": 390 + }, + { + "clip_ratio": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 298.75, + "completions/max_terminated_length": 298.75, + "completions/mean_length": 199.359375, + "completions/mean_terminated_length": 199.359375, + "completions/min_length": 133.325, + "completions/min_terminated_length": 133.325, + "epoch": 0.6964954815957681, + "grad_norm": 0.36379488564541623, + "kl": 0.023514556884765624, + "learning_rate": 1e-06, + "loss": 0.0139, + "num_tokens": 59453574.0, + "reward": 0.7852070070803165, + "reward_std": 0.11039148237323389, + "rewards/format_reward/mean": 1.0, + "rewards/format_reward/std": 0.0, + "rewards/qatch_metrics/mean": 0.7473023503087461, + "rewards/qatch_metrics/std": 0.12987233807798476, + "rewards/tag_count_reward/mean": 1.0, + "rewards/tag_count_reward/std": 0.0, + "step": 395 + }, + { + "clip_ratio": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 297.525, + "completions/max_terminated_length": 297.525, + "completions/mean_length": 192.63515625, + "completions/mean_terminated_length": 192.63515625, + "completions/min_length": 130.4375, + "completions/min_terminated_length": 130.4375, + "epoch": 0.7053118800969804, + "grad_norm": 0.31515335460233057, + "kl": 0.02534027099609375, + "learning_rate": 1e-06, + "loss": -0.0013, + "num_tokens": 60164595.0, + "reward": 0.8131733348593115, + "reward_std": 0.09474163451232016, + "rewards/format_reward/mean": 1.0, + "rewards/format_reward/std": 0.0, + "rewards/qatch_metrics/mean": 0.7802039116621018, + "rewards/qatch_metrics/std": 0.11146075297147036, + "rewards/tag_count_reward/mean": 1.0, + "rewards/tag_count_reward/std": 0.0, + "step": 400 + }, + { + "clip_ratio": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 301.3375, + "completions/max_terminated_length": 301.3375, + "completions/mean_length": 194.86171875, + "completions/mean_terminated_length": 194.86171875, + "completions/min_length": 132.875, + "completions/min_terminated_length": 132.875, + "epoch": 0.7141282785981926, + "grad_norm": 0.36020622913136857, + "kl": 0.02452239990234375, + "learning_rate": 1e-06, + "loss": -0.0044, + "num_tokens": 60876834.0, + "reward": 0.7978186447173357, + "reward_std": 0.12316365442238748, + "rewards/format_reward/mean": 0.99921875, + "rewards/format_reward/std": 0.003125, + "rewards/qatch_metrics/mean": 0.7622429746668786, + "rewards/qatch_metrics/std": 0.14448482398875057, + "rewards/tag_count_reward/mean": 0.9998046875, + "rewards/tag_count_reward/std": 0.00078125, + "step": 405 + }, + { + "clip_ratio": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 330.2875, + "completions/max_terminated_length": 330.2875, + "completions/mean_length": 222.946875, + "completions/mean_terminated_length": 222.946875, + "completions/min_length": 147.725, + "completions/min_terminated_length": 147.725, + "epoch": 0.7229446770994049, + "grad_norm": 0.23920846142153848, + "kl": 0.022620391845703126, + "learning_rate": 1e-06, + "loss": 0.0015, + "num_tokens": 61628878.0, + "reward": 0.8119752595201135, + "reward_std": 0.09445713473833166, + "rewards/format_reward/mean": 1.0, + "rewards/format_reward/std": 0.0, + "rewards/qatch_metrics/mean": 0.7788403697311879, + "rewards/qatch_metrics/std": 0.11107292836531997, + "rewards/tag_count_reward/mean": 0.99921875, + "rewards/tag_count_reward/std": 0.003125, + "step": 410 + }, + { + "clip_ratio": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 362.675, + "completions/max_terminated_length": 362.675, + "completions/mean_length": 238.16640625, + "completions/mean_terminated_length": 238.16640625, + "completions/min_length": 149.55, + "completions/min_terminated_length": 149.55, + "epoch": 0.7317610756006171, + "grad_norm": 0.29869110706075575, + "kl": 0.02274169921875, + "learning_rate": 1e-06, + "loss": 0.0076, + "num_tokens": 62407347.0, + "reward": 0.8343930047005415, + "reward_std": 0.1020151567645371, + "rewards/format_reward/mean": 1.0, + "rewards/format_reward/std": 0.0, + "rewards/qatch_metrics/mean": 0.805168230831623, + "rewards/qatch_metrics/std": 0.12001783675514162, + "rewards/tag_count_reward/mean": 1.0, + "rewards/tag_count_reward/std": 0.0, + "step": 415 + }, + { + "clip_ratio": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 351.1625, + "completions/max_terminated_length": 351.1625, + "completions/mean_length": 238.10625, + "completions/mean_terminated_length": 238.10625, + "completions/min_length": 151.7, + "completions/min_terminated_length": 151.7, + "epoch": 0.7405774741018294, + "grad_norm": 0.2830612988897095, + "kl": 0.021949005126953126, + "learning_rate": 1e-06, + "loss": 0.0002, + "num_tokens": 63172059.0, + "reward": 0.82713833283633, + "reward_std": 0.07888575517572463, + "rewards/format_reward/mean": 0.99921875, + "rewards/format_reward/std": 0.003125, + "rewards/qatch_metrics/mean": 0.7967367230914533, + "rewards/qatch_metrics/std": 0.0928413406247273, + "rewards/tag_count_reward/mean": 0.9998046875, + "rewards/tag_count_reward/std": 0.00078125, + "step": 420 + }, + { + "clip_ratio": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 350.6625, + "completions/max_terminated_length": 350.6625, + "completions/mean_length": 234.71875, + "completions/mean_terminated_length": 234.71875, + "completions/min_length": 152.85, + "completions/min_terminated_length": 152.85, + "epoch": 0.7493938726030417, + "grad_norm": 0.303175281797131, + "kl": 0.023668670654296876, + "learning_rate": 1e-06, + "loss": 0.0014, + "num_tokens": 63958371.0, + "reward": 0.8379335630685091, + "reward_std": 0.08907212568446994, + "rewards/format_reward/mean": 1.0, + "rewards/format_reward/std": 0.0, + "rewards/qatch_metrics/mean": 0.8093335981713607, + "rewards/qatch_metrics/std": 0.10479074087925255, + "rewards/tag_count_reward/mean": 1.0, + "rewards/tag_count_reward/std": 0.0, + "step": 425 + }, + { + "clip_ratio": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 356.15, + "completions/max_terminated_length": 356.15, + "completions/mean_length": 233.284375, + "completions/mean_terminated_length": 233.284375, + "completions/min_length": 144.625, + "completions/min_terminated_length": 144.625, + "epoch": 0.7582102711042539, + "grad_norm": 0.3362377001779979, + "kl": 0.020977020263671875, + "learning_rate": 1e-06, + "loss": 0.0094, + "num_tokens": 64755503.0, + "reward": 0.7857262346893549, + "reward_std": 0.0779630596237439, + "rewards/format_reward/mean": 0.99921875, + "rewards/format_reward/std": 0.003125, + "rewards/qatch_metrics/mean": 0.748039587616222, + "rewards/qatch_metrics/std": 0.09121573810989503, + "rewards/tag_count_reward/mean": 0.9994140625, + "rewards/tag_count_reward/std": 0.00234375, + "step": 430 + }, + { + "clip_ratio": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 342.0875, + "completions/max_terminated_length": 342.0875, + "completions/mean_length": 224.60390625, + "completions/mean_terminated_length": 224.60390625, + "completions/min_length": 139.6875, + "completions/min_terminated_length": 139.6875, + "epoch": 0.7670266696054662, + "grad_norm": 0.27147477635209977, + "kl": 0.023795318603515626, + "learning_rate": 1e-06, + "loss": 0.007, + "num_tokens": 65500036.0, + "reward": 0.8154280468821525, + "reward_std": 0.0885659102234058, + "rewards/format_reward/mean": 1.0, + "rewards/format_reward/std": 0.0, + "rewards/qatch_metrics/mean": 0.7828565156087279, + "rewards/qatch_metrics/std": 0.10419519691495224, + "rewards/tag_count_reward/mean": 1.0, + "rewards/tag_count_reward/std": 0.0, + "step": 435 + }, + { + "clip_ratio": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 343.15, + "completions/max_terminated_length": 343.15, + "completions/mean_length": 225.971875, + "completions/mean_terminated_length": 225.971875, + "completions/min_length": 134.8875, + "completions/min_terminated_length": 134.8875, + "epoch": 0.7758430681066785, + "grad_norm": 0.23493992268435515, + "kl": 0.0210540771484375, + "learning_rate": 1e-06, + "loss": 0.001, + "num_tokens": 66252064.0, + "reward": 0.8585389815270901, + "reward_std": 0.08183195294986945, + "rewards/format_reward/mean": 1.0, + "rewards/format_reward/std": 0.0, + "rewards/qatch_metrics/mean": 0.8335752619896084, + "rewards/qatch_metrics/std": 0.09627289194759214, + "rewards/tag_count_reward/mean": 1.0, + "rewards/tag_count_reward/std": 0.0, + "step": 440 + }, + { + "clip_ratio": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 355.575, + "completions/max_terminated_length": 355.575, + "completions/mean_length": 242.10859375, + "completions/mean_terminated_length": 242.10859375, + "completions/min_length": 151.175, + "completions/min_terminated_length": 151.175, + "epoch": 0.7846594666078907, + "grad_norm": 0.2140376472128798, + "kl": 0.02173614501953125, + "learning_rate": 1e-06, + "loss": 0.0003, + "num_tokens": 67023275.0, + "reward": 0.87310497071594, + "reward_std": 0.0631409589201212, + "rewards/format_reward/mean": 1.0, + "rewards/format_reward/std": 0.0, + "rewards/qatch_metrics/mean": 0.8507117207162083, + "rewards/qatch_metrics/std": 0.07428348822286353, + "rewards/tag_count_reward/mean": 1.0, + "rewards/tag_count_reward/std": 0.0, + "step": 445 + }, + { + "clip_ratio": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 353.0, + "completions/max_terminated_length": 353.0, + "completions/mean_length": 226.27578125, + "completions/mean_terminated_length": 226.27578125, + "completions/min_length": 139.95, + "completions/min_terminated_length": 139.95, + "epoch": 0.793475865109103, + "grad_norm": 0.3122440505453943, + "kl": 0.021511077880859375, + "learning_rate": 1e-06, + "loss": 0.0005, + "num_tokens": 67777324.0, + "reward": 0.8520105175673962, + "reward_std": 0.08210055120289325, + "rewards/format_reward/mean": 0.99921875, + "rewards/format_reward/std": 0.003125, + "rewards/qatch_metrics/mean": 0.8260210990672932, + "rewards/qatch_metrics/std": 0.09612429473781958, + "rewards/tag_count_reward/mean": 0.9994140625, + "rewards/tag_count_reward/std": 0.00234375, + "step": 450 + }, + { + "clip_ratio": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 327.425, + "completions/max_terminated_length": 327.425, + "completions/mean_length": 212.48125, + "completions/mean_terminated_length": 212.48125, + "completions/min_length": 135.5125, + "completions/min_terminated_length": 135.5125, + "epoch": 0.8022922636103151, + "grad_norm": 0.3121983142519688, + "kl": 0.022968292236328125, + "learning_rate": 1e-06, + "loss": 0.0106, + "num_tokens": 68512644.0, + "reward": 0.8407730983570219, + "reward_std": 0.06822409054730087, + "rewards/format_reward/mean": 1.0, + "rewards/format_reward/std": 0.0, + "rewards/qatch_metrics/mean": 0.8126742199849104, + "rewards/qatch_metrics/std": 0.08026364156394265, + "rewards/tag_count_reward/mean": 1.0, + "rewards/tag_count_reward/std": 0.0, + "step": 455 + }, + { + "clip_ratio": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 340.25, + "completions/max_terminated_length": 340.25, + "completions/mean_length": 223.44375, + "completions/mean_terminated_length": 223.44375, + "completions/min_length": 141.9625, + "completions/min_terminated_length": 141.9625, + "epoch": 0.8111086621115274, + "grad_norm": 0.3458299809901387, + "kl": 0.02670440673828125, + "learning_rate": 1e-06, + "loss": -0.0033, + "num_tokens": 69262636.0, + "reward": 0.8292963147163391, + "reward_std": 0.08730166773311794, + "rewards/format_reward/mean": 0.99921875, + "rewards/format_reward/std": 0.003125, + "rewards/qatch_metrics/mean": 0.7992755254730582, + "rewards/qatch_metrics/std": 0.10261548004345969, + "rewards/tag_count_reward/mean": 0.9998046875, + "rewards/tag_count_reward/std": 0.00078125, + "step": 460 + }, + { + "clip_ratio": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 340.2375, + "completions/max_terminated_length": 340.2375, + "completions/mean_length": 223.1640625, + "completions/mean_terminated_length": 223.1640625, + "completions/min_length": 144.9375, + "completions/min_terminated_length": 144.9375, + "epoch": 0.8199250606127397, + "grad_norm": 0.2539455340605116, + "kl": 0.02646636962890625, + "learning_rate": 1e-06, + "loss": 0.0074, + "num_tokens": 70028046.0, + "reward": 0.782775210775435, + "reward_std": 0.09664202006024425, + "rewards/format_reward/mean": 1.0, + "rewards/format_reward/std": 0.0, + "rewards/qatch_metrics/mean": 0.7444414100635186, + "rewards/qatch_metrics/std": 0.11369650048291077, + "rewards/tag_count_reward/mean": 1.0, + "rewards/tag_count_reward/std": 0.0, + "step": 465 + }, + { + "clip_ratio": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 343.65, + "completions/max_terminated_length": 343.65, + "completions/mean_length": 220.91953125, + "completions/mean_terminated_length": 220.91953125, + "completions/min_length": 142.9125, + "completions/min_terminated_length": 142.9125, + "epoch": 0.8287414591139519, + "grad_norm": 0.29340586589965106, + "kl": 0.023850250244140624, + "learning_rate": 1e-06, + "loss": 0.0028, + "num_tokens": 70817127.0, + "reward": 0.83585394769907, + "reward_std": 0.08243265537312254, + "rewards/format_reward/mean": 1.0, + "rewards/format_reward/std": 0.0, + "rewards/qatch_metrics/mean": 0.8068869862705469, + "rewards/qatch_metrics/std": 0.09697959922486916, + "rewards/tag_count_reward/mean": 1.0, + "rewards/tag_count_reward/std": 0.0, + "step": 470 + }, + { + "clip_ratio": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 319.25, + "completions/max_terminated_length": 319.25, + "completions/mean_length": 212.4921875, + "completions/mean_terminated_length": 212.4921875, + "completions/min_length": 137.675, + "completions/min_terminated_length": 137.675, + "epoch": 0.8375578576151642, + "grad_norm": 0.30646618504129836, + "kl": 0.02505950927734375, + "learning_rate": 1e-06, + "loss": -0.0023, + "num_tokens": 71553357.0, + "reward": 0.8542880930006505, + "reward_std": 0.07209075510618276, + "rewards/format_reward/mean": 1.0, + "rewards/format_reward/std": 0.0, + "rewards/qatch_metrics/mean": 0.8285742185791605, + "rewards/qatch_metrics/std": 0.08481265990703832, + "rewards/tag_count_reward/mean": 1.0, + "rewards/tag_count_reward/std": 0.0, + "step": 475 + }, + { + "clip_ratio": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 342.7375, + "completions/max_terminated_length": 342.7375, + "completions/mean_length": 233.45546875, + "completions/mean_terminated_length": 233.45546875, + "completions/min_length": 147.2125, + "completions/min_terminated_length": 147.2125, + "epoch": 0.8463742561163765, + "grad_norm": 0.3162519376309847, + "kl": 0.0266021728515625, + "learning_rate": 1e-06, + "loss": 0.0054, + "num_tokens": 72361556.0, + "reward": 0.8315023425966501, + "reward_std": 0.10010532954183873, + "rewards/format_reward/mean": 1.0, + "rewards/format_reward/std": 0.0, + "rewards/qatch_metrics/mean": 0.8017674525035545, + "rewards/qatch_metrics/std": 0.11777098168386146, + "rewards/tag_count_reward/mean": 1.0, + "rewards/tag_count_reward/std": 0.0, + "step": 480 + }, + { + "clip_ratio": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 341.25, + "completions/max_terminated_length": 341.25, + "completions/mean_length": 234.0140625, + "completions/mean_terminated_length": 234.0140625, + "completions/min_length": 154.7, + "completions/min_terminated_length": 154.7, + "epoch": 0.8551906546175887, + "grad_norm": 0.297834941840183, + "kl": 0.02375946044921875, + "learning_rate": 1e-06, + "loss": 0.0113, + "num_tokens": 73116902.0, + "reward": 0.7931279448792339, + "reward_std": 0.08709526733728126, + "rewards/format_reward/mean": 1.0, + "rewards/format_reward/std": 0.0, + "rewards/qatch_metrics/mean": 0.7566210999619216, + "rewards/qatch_metrics/std": 0.10246502548689022, + "rewards/tag_count_reward/mean": 1.0, + "rewards/tag_count_reward/std": 0.0, + "step": 485 + }, + { + "clip_ratio": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 324.5625, + "completions/max_terminated_length": 324.5625, + "completions/mean_length": 220.03203125, + "completions/mean_terminated_length": 220.03203125, + "completions/min_length": 138.8125, + "completions/min_terminated_length": 138.8125, + "epoch": 0.864007053118801, + "grad_norm": 0.20060310802569065, + "kl": 0.022344970703125, + "learning_rate": 1e-06, + "loss": 0.0071, + "num_tokens": 73868767.0, + "reward": 0.8470947509631515, + "reward_std": 0.06845701420679688, + "rewards/format_reward/mean": 1.0, + "rewards/format_reward/std": 0.0, + "rewards/qatch_metrics/mean": 0.8201114618219435, + "rewards/qatch_metrics/std": 0.08053766970988363, + "rewards/tag_count_reward/mean": 1.0, + "rewards/tag_count_reward/std": 0.0, + "step": 490 + }, + { + "clip_ratio": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 315.8, + "completions/max_terminated_length": 315.8, + "completions/mean_length": 211.5171875, + "completions/mean_terminated_length": 211.5171875, + "completions/min_length": 139.7625, + "completions/min_terminated_length": 139.7625, + "epoch": 0.8728234516200132, + "grad_norm": 0.2914817614366052, + "kl": 0.023822784423828125, + "learning_rate": 1e-06, + "loss": -0.0034, + "num_tokens": 74608789.0, + "reward": 0.8249077584594489, + "reward_std": 0.08015898242010736, + "rewards/format_reward/mean": 1.0, + "rewards/format_reward/std": 0.0, + "rewards/qatch_metrics/mean": 0.7940091195050627, + "rewards/qatch_metrics/std": 0.09430469113285653, + "rewards/tag_count_reward/mean": 1.0, + "rewards/tag_count_reward/std": 0.0, + "step": 495 + }, + { + "clip_ratio": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 330.7, + "completions/max_terminated_length": 330.7, + "completions/mean_length": 225.6296875, + "completions/mean_terminated_length": 225.6296875, + "completions/min_length": 147.9125, + "completions/min_terminated_length": 147.9125, + "epoch": 0.8816398501212255, + "grad_norm": 0.27724680534940715, + "kl": 0.024609375, + "learning_rate": 1e-06, + "loss": 0.0044, + "num_tokens": 75370139.0, + "reward": 0.7901407666504383, + "reward_std": 0.08103454456286271, + "rewards/format_reward/mean": 1.0, + "rewards/format_reward/std": 0.0, + "rewards/qatch_metrics/mean": 0.7531067751348018, + "rewards/qatch_metrics/std": 0.09533476178003183, + "rewards/tag_count_reward/mean": 1.0, + "rewards/tag_count_reward/std": 0.0, + "step": 500 + }, + { + "clip_ratio": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 333.875, + "completions/max_terminated_length": 333.875, + "completions/mean_length": 220.46640625, + "completions/mean_terminated_length": 220.46640625, + "completions/min_length": 142.2, + "completions/min_terminated_length": 142.2, + "epoch": 0.8904562486224378, + "grad_norm": 0.23661752197680214, + "kl": 0.022271728515625, + "learning_rate": 1e-06, + "loss": 0.0066, + "num_tokens": 76121648.0, + "reward": 0.8500555850565433, + "reward_std": 0.07330139055848121, + "rewards/format_reward/mean": 1.0, + "rewards/format_reward/std": 0.0, + "rewards/qatch_metrics/mean": 0.8235947942361236, + "rewards/qatch_metrics/std": 0.08623693531844764, + "rewards/tag_count_reward/mean": 1.0, + "rewards/tag_count_reward/std": 0.0, + "step": 505 + }, + { + "clip_ratio": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 317.7875, + "completions/max_terminated_length": 317.7875, + "completions/mean_length": 210.9671875, + "completions/mean_terminated_length": 210.9671875, + "completions/min_length": 138.975, + "completions/min_terminated_length": 138.975, + "epoch": 0.89927264712365, + "grad_norm": 0.21851493451825393, + "kl": 0.025273895263671874, + "learning_rate": 1e-06, + "loss": 0.0036, + "num_tokens": 76872790.0, + "reward": 0.8583780597895384, + "reward_std": 0.08067860676383135, + "rewards/format_reward/mean": 1.0, + "rewards/format_reward/std": 0.0, + "rewards/qatch_metrics/mean": 0.8333859436213971, + "rewards/qatch_metrics/std": 0.09491601307672681, + "rewards/tag_count_reward/mean": 1.0, + "rewards/tag_count_reward/std": 0.0, + "step": 510 + }, + { + "clip_ratio": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 321.775, + "completions/max_terminated_length": 321.775, + "completions/mean_length": 216.0375, + "completions/mean_terminated_length": 216.0375, + "completions/min_length": 141.1625, + "completions/min_terminated_length": 141.1625, + "epoch": 0.9080890456248623, + "grad_norm": 0.2829422652691582, + "kl": 0.025566864013671874, + "learning_rate": 1e-06, + "loss": 0.0082, + "num_tokens": 77616342.0, + "reward": 0.8006692606955766, + "reward_std": 0.08726950597483665, + "rewards/format_reward/mean": 1.0, + "rewards/format_reward/std": 0.0, + "rewards/qatch_metrics/mean": 0.7654932357370854, + "rewards/qatch_metrics/std": 0.10267001276370138, + "rewards/tag_count_reward/mean": 1.0, + "rewards/tag_count_reward/std": 0.0, + "step": 515 + }, + { + "clip_ratio": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 290.275, + "completions/max_terminated_length": 290.275, + "completions/mean_length": 195.296875, + "completions/mean_terminated_length": 195.296875, + "completions/min_length": 130.9125, + "completions/min_terminated_length": 130.9125, + "epoch": 0.9169054441260746, + "grad_norm": 0.2228119611592202, + "kl": 0.023336029052734374, + "learning_rate": 1e-06, + "loss": 0.0065, + "num_tokens": 78313506.0, + "reward": 0.7732636205852031, + "reward_std": 0.05938901338377036, + "rewards/format_reward/mean": 1.0, + "rewards/format_reward/std": 0.0, + "rewards/qatch_metrics/mean": 0.7332513069733977, + "rewards/qatch_metrics/std": 0.06986943007213994, + "rewards/tag_count_reward/mean": 1.0, + "rewards/tag_count_reward/std": 0.0, + "step": 520 + }, + { + "clip_ratio": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 305.6125, + "completions/max_terminated_length": 305.6125, + "completions/mean_length": 205.165625, + "completions/mean_terminated_length": 205.165625, + "completions/min_length": 137.2, + "completions/min_terminated_length": 137.2, + "epoch": 0.9257218426272867, + "grad_norm": 0.36003445762352576, + "kl": 0.023264312744140626, + "learning_rate": 1e-06, + "loss": 0.0079, + "num_tokens": 79054486.0, + "reward": 0.8107579160481692, + "reward_std": 0.07336599697882776, + "rewards/format_reward/mean": 1.0, + "rewards/format_reward/std": 0.0, + "rewards/qatch_metrics/mean": 0.7773622425273061, + "rewards/qatch_metrics/std": 0.08631294009974226, + "rewards/tag_count_reward/mean": 1.0, + "rewards/tag_count_reward/std": 0.0, + "step": 525 + }, + { + "clip_ratio": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 323.2125, + "completions/max_terminated_length": 323.2125, + "completions/mean_length": 220.25, + "completions/mean_terminated_length": 220.25, + "completions/min_length": 145.8625, + "completions/min_terminated_length": 145.8625, + "epoch": 0.934538241128499, + "grad_norm": 0.30049379873568555, + "kl": 0.0247039794921875, + "learning_rate": 1e-06, + "loss": 0.0046, + "num_tokens": 79821798.0, + "reward": 0.8396705185994506, + "reward_std": 0.07967656154651195, + "rewards/format_reward/mean": 0.99921875, + "rewards/format_reward/std": 0.003125, + "rewards/qatch_metrics/mean": 0.811480475217104, + "rewards/qatch_metrics/std": 0.09332353379577399, + "rewards/tag_count_reward/mean": 0.9998046875, + "rewards/tag_count_reward/std": 0.00078125, + "step": 530 + }, + { + "clip_ratio": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 334.6, + "completions/max_terminated_length": 334.6, + "completions/mean_length": 224.428125, + "completions/mean_terminated_length": 224.428125, + "completions/min_length": 143.85, + "completions/min_terminated_length": 143.85, + "epoch": 0.9433546396297112, + "grad_norm": 0.26616721247221914, + "kl": 0.024881744384765626, + "learning_rate": 1e-06, + "loss": 0.0114, + "num_tokens": 80588906.0, + "reward": 0.8039220564067364, + "reward_std": 0.08752607379574329, + "rewards/format_reward/mean": 1.0, + "rewards/format_reward/std": 0.0, + "rewards/qatch_metrics/mean": 0.7693200555630029, + "rewards/qatch_metrics/std": 0.10297185693052598, + "rewards/tag_count_reward/mean": 1.0, + "rewards/tag_count_reward/std": 0.0, + "step": 535 + }, + { + "clip_ratio": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 337.025, + "completions/max_terminated_length": 337.025, + "completions/mean_length": 222.8765625, + "completions/mean_terminated_length": 222.8765625, + "completions/min_length": 146.35, + "completions/min_terminated_length": 146.35, + "epoch": 0.9521710381309235, + "grad_norm": 0.26056623342737706, + "kl": 0.023919677734375, + "learning_rate": 1e-06, + "loss": 0.0073, + "num_tokens": 81348428.0, + "reward": 0.8161520700901747, + "reward_std": 0.06797666533384472, + "rewards/format_reward/mean": 0.9984375, + "rewards/format_reward/std": 0.00625, + "rewards/qatch_metrics/mean": 0.7839151054620743, + "rewards/qatch_metrics/std": 0.07955915104830638, + "rewards/tag_count_reward/mean": 0.999609375, + "rewards/tag_count_reward/std": 0.0015625, + "step": 540 + }, + { + "clip_ratio": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 329.45, + "completions/max_terminated_length": 329.45, + "completions/mean_length": 222.23515625, + "completions/mean_terminated_length": 222.23515625, + "completions/min_length": 145.9375, + "completions/min_terminated_length": 145.9375, + "epoch": 0.9609874366321358, + "grad_norm": 0.25072778848110355, + "kl": 0.02654876708984375, + "learning_rate": 1e-06, + "loss": 0.0051, + "num_tokens": 82112537.0, + "reward": 0.8805699178948998, + "reward_std": 0.07740147057920695, + "rewards/format_reward/mean": 1.0, + "rewards/format_reward/std": 0.0, + "rewards/qatch_metrics/mean": 0.8594940162263811, + "rewards/qatch_metrics/std": 0.09106055875308812, + "rewards/tag_count_reward/mean": 1.0, + "rewards/tag_count_reward/std": 0.0, + "step": 545 + }, + { + "clip_ratio": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 326.8125, + "completions/max_terminated_length": 326.8125, + "completions/mean_length": 216.265625, + "completions/mean_terminated_length": 216.265625, + "completions/min_length": 142.5, + "completions/min_terminated_length": 142.5, + "epoch": 0.969803835133348, + "grad_norm": 0.25948373422406573, + "kl": 0.024462890625, + "learning_rate": 1e-06, + "loss": 0.0006, + "num_tokens": 82872173.0, + "reward": 0.8117451569065451, + "reward_std": 0.0860868067946285, + "rewards/format_reward/mean": 1.0, + "rewards/format_reward/std": 0.0, + "rewards/qatch_metrics/mean": 0.7785237034782767, + "rewards/qatch_metrics/std": 0.10127860223874449, + "rewards/tag_count_reward/mean": 1.0, + "rewards/tag_count_reward/std": 0.0, + "step": 550 + }, + { + "clip_ratio": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 337.1875, + "completions/max_terminated_length": 337.1875, + "completions/mean_length": 216.21640625, + "completions/mean_terminated_length": 216.21640625, + "completions/min_length": 137.8125, + "completions/min_terminated_length": 137.8125, + "epoch": 0.9786202336345603, + "grad_norm": 0.3645941597990175, + "kl": 0.02496337890625, + "learning_rate": 1e-06, + "loss": -0.0009, + "num_tokens": 83634770.0, + "reward": 0.8101593714207411, + "reward_std": 0.08605160953738959, + "rewards/format_reward/mean": 1.0, + "rewards/format_reward/std": 0.0, + "rewards/qatch_metrics/mean": 0.7766580768162384, + "rewards/qatch_metrics/std": 0.10123719241237268, + "rewards/tag_count_reward/mean": 1.0, + "rewards/tag_count_reward/std": 0.0, + "step": 555 + }, + { + "clip_ratio": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 343.25, + "completions/max_terminated_length": 343.25, + "completions/mean_length": 231.0015625, + "completions/mean_terminated_length": 231.0015625, + "completions/min_length": 151.325, + "completions/min_terminated_length": 151.325, + "epoch": 0.9874366321357725, + "grad_norm": 0.21758575977885092, + "kl": 0.023052978515625, + "learning_rate": 1e-06, + "loss": 0.0066, + "num_tokens": 84398692.0, + "reward": 0.8315089832991361, + "reward_std": 0.07600237202714197, + "rewards/format_reward/mean": 1.0, + "rewards/format_reward/std": 0.0, + "rewards/qatch_metrics/mean": 0.8017752626910806, + "rewards/qatch_metrics/std": 0.08941456201137044, + "rewards/tag_count_reward/mean": 1.0, + "rewards/tag_count_reward/std": 0.0, + "step": 560 + }, + { + "clip_ratio": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 355.1375, + "completions/max_terminated_length": 355.1375, + "completions/mean_length": 232.215625, + "completions/mean_terminated_length": 232.215625, + "completions/min_length": 148.6875, + "completions/min_terminated_length": 148.6875, + "epoch": 0.9962530306369848, + "grad_norm": 0.2378336659692683, + "kl": 0.023166656494140625, + "learning_rate": 1e-06, + "loss": 0.0083, + "num_tokens": 85155592.0, + "reward": 0.7450484920293092, + "reward_std": 0.07769823344424368, + "rewards/format_reward/mean": 1.0, + "rewards/format_reward/std": 0.0, + "rewards/qatch_metrics/mean": 0.7000570335425437, + "rewards/qatch_metrics/std": 0.09140969021318597, + "rewards/tag_count_reward/mean": 1.0, + "rewards/tag_count_reward/std": 0.0, + "step": 565 + }, + { + "clip_ratio": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 290.0625, + "completions/max_terminated_length": 290.0625, + "completions/mean_length": 187.46484375, + "completions/mean_terminated_length": 187.46484375, + "completions/min_length": 124.65625, + "completions/min_terminated_length": 124.65625, + "epoch": 0.9997795900374697, + "kl": 0.027322769165039062, + "num_tokens": 85427878.0, + "reward": 0.8777023833245039, + "reward_std": 0.08281319939123932, + "rewards/format_reward/mean": 1.0, + "rewards/format_reward/std": 0.0, + "rewards/qatch_metrics/mean": 0.8561204457655549, + "rewards/qatch_metrics/std": 0.09742730065772776, + "rewards/tag_count_reward/mean": 1.0, + "rewards/tag_count_reward/std": 0.0, + "step": 567, + "total_flos": 0.0, + "train_loss": 0.0013445673257480432, + "train_runtime": 158207.651, + "train_samples_per_second": 0.057, + "train_steps_per_second": 0.004 + } + ], + "logging_steps": 5, + "max_steps": 567, + "num_input_tokens_seen": 85427878, + "num_train_epochs": 1, + "save_steps": 57, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": true + }, + "attributes": {} + } + }, + "total_flos": 0.0, + "train_batch_size": 8, + "trial_name": null, + "trial_params": null +}