diff --git "a/trainer_state.json" "b/trainer_state.json"
new file mode 100644--- /dev/null
+++ "b/trainer_state.json"
@@ -0,0 +1,3136 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 0.9982859101816935,
+  "eval_steps": 0,
+  "global_step": 182,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.005485087418580734,
+      "grad_norm": 5.038297822406707,
+      "learning_rate": 2.6315789473684213e-07,
+      "logits/chosen": -0.3854110836982727,
+      "logits/rejected": -0.38843637704849243,
+      "logps/chosen": -0.5867404937744141,
+      "logps/rejected": -0.7349259853363037,
+      "loss": 0.8549,
+      "odds_ratio_loss": 8.495767593383789,
+      "rewards/accuracies": 0.328125,
+      "rewards/chosen": -0.07349259406328201,
+      "rewards/margins": -0.014818555675446987,
+      "rewards/rejected": -0.05867404490709305,
+      "sft_loss": 0.00530597660690546,
+      "step": 1
+    },
+    {
+      "epoch": 0.010970174837161468,
+      "grad_norm": 3.8688701991936516,
+      "learning_rate": 5.263157894736843e-07,
+      "logits/chosen": -0.4200110137462616,
+      "logits/rejected": -0.4337027370929718,
+      "logps/chosen": -0.5888247489929199,
+      "logps/rejected": -0.7141146659851074,
+      "loss": 0.8261,
+      "odds_ratio_loss": 8.218369483947754,
+      "rewards/accuracies": 0.3671875,
+      "rewards/chosen": -0.07141146808862686,
+      "rewards/margins": -0.01252899132668972,
+      "rewards/rejected": -0.05888247489929199,
+      "sft_loss": 0.004305172245949507,
+      "step": 2
+    },
+    {
+      "epoch": 0.0164552622557422,
+      "grad_norm": 3.4925619509791903,
+      "learning_rate": 7.894736842105263e-07,
+      "logits/chosen": -0.38829293847084045,
+      "logits/rejected": -0.36370420455932617,
+      "logps/chosen": -0.6846970319747925,
+      "logps/rejected": -0.6908957362174988,
+      "loss": 0.8057,
+      "odds_ratio_loss": 8.025218963623047,
+      "rewards/accuracies": 0.375,
+      "rewards/chosen": -0.06908956915140152,
+      "rewards/margins": -0.0006198745686560869,
+      "rewards/rejected": -0.06846970319747925,
+      "sft_loss": 0.003192172385752201,
+      "step": 3
+    },
+    {
+      "epoch": 0.021940349674322936,
+      "grad_norm": 5.442119869368371,
+      "learning_rate": 1.0526315789473685e-06,
+      "logits/chosen": -0.4157654047012329,
+      "logits/rejected": -0.4449327886104584,
+      "logps/chosen": -0.6106168031692505,
+      "logps/rejected": -0.7167029976844788,
+      "loss": 0.8331,
+      "odds_ratio_loss": 8.275010108947754,
+      "rewards/accuracies": 0.4140625,
+      "rewards/chosen": -0.0716702938079834,
+      "rewards/margins": -0.010608619078993797,
+      "rewards/rejected": -0.06106168404221535,
+      "sft_loss": 0.005572030786424875,
+      "step": 4
+    },
+    {
+      "epoch": 0.027425437092903668,
+      "grad_norm": 3.9040280718139004,
+      "learning_rate": 1.3157894736842106e-06,
+      "logits/chosen": -0.4705536961555481,
+      "logits/rejected": -0.448209285736084,
+      "logps/chosen": -0.5760213136672974,
+      "logps/rejected": -0.6794447898864746,
+      "loss": 0.7994,
+      "odds_ratio_loss": 7.961273193359375,
+      "rewards/accuracies": 0.3359375,
+      "rewards/chosen": -0.0679444819688797,
+      "rewards/margins": -0.010342349298298359,
+      "rewards/rejected": -0.057602137327194214,
+      "sft_loss": 0.003320502582937479,
+      "step": 5
+    },
+    {
+      "epoch": 0.0329105245114844,
+      "grad_norm": 3.9490673466980417,
+      "learning_rate": 1.5789473684210526e-06,
+      "logits/chosen": -0.3221694231033325,
+      "logits/rejected": -0.405836820602417,
+      "logps/chosen": -0.6044615507125854,
+      "logps/rejected": -0.7186299562454224,
+      "loss": 0.8315,
+      "odds_ratio_loss": 8.270588874816895,
+      "rewards/accuracies": 0.375,
+      "rewards/chosen": -0.07186298817396164,
+      "rewards/margins": -0.011416830122470856,
+      "rewards/rejected": -0.060446158051490784,
+      "sft_loss": 0.004453588742762804,
+      "step": 6
+    },
+    {
+      "epoch": 0.03839561193006513,
+      "grad_norm": 4.211003798628622,
+      "learning_rate": 1.8421052631578948e-06,
+      "logits/chosen": -0.4302704334259033,
+      "logits/rejected": -0.48908236622810364,
+      "logps/chosen": -0.6016876697540283,
+      "logps/rejected": -0.6534877419471741,
+      "loss": 0.7601,
+      "odds_ratio_loss": 7.552478313446045,
+      "rewards/accuracies": 0.4140625,
+      "rewards/chosen": -0.06534876674413681,
+      "rewards/margins": -0.005180003587156534,
+      "rewards/rejected": -0.06016876921057701,
+      "sft_loss": 0.004846740514039993,
+      "step": 7
+    },
+    {
+      "epoch": 0.04388069934864587,
+      "grad_norm": 4.5970681547716055,
+      "learning_rate": 2.105263157894737e-06,
+      "logits/chosen": -0.4125446677207947,
+      "logits/rejected": -0.48786473274230957,
+      "logps/chosen": -0.5902884602546692,
+      "logps/rejected": -0.6903232336044312,
+      "loss": 0.8077,
+      "odds_ratio_loss": 8.029266357421875,
+      "rewards/accuracies": 0.390625,
+      "rewards/chosen": -0.06903232634067535,
+      "rewards/margins": -0.010003475472331047,
+      "rewards/rejected": -0.05902884528040886,
+      "sft_loss": 0.004780109040439129,
+      "step": 8
+    },
+    {
+      "epoch": 0.049365786767226603,
+      "grad_norm": 3.9830322243241154,
+      "learning_rate": 2.368421052631579e-06,
+      "logits/chosen": -0.4158952832221985,
+      "logits/rejected": -0.4516904354095459,
+      "logps/chosen": -0.49427568912506104,
+      "logps/rejected": -0.7636269927024841,
+      "loss": 0.8995,
+      "odds_ratio_loss": 8.943946838378906,
+      "rewards/accuracies": 0.265625,
+      "rewards/chosen": -0.07636269927024841,
+      "rewards/margins": -0.02693513222038746,
+      "rewards/rejected": -0.049427565187215805,
+      "sft_loss": 0.005144394934177399,
+      "step": 9
+    },
+    {
+      "epoch": 0.054850874185807336,
+      "grad_norm": 3.963656319575701,
+      "learning_rate": 2.631578947368421e-06,
+      "logits/chosen": -0.4935796558856964,
+      "logits/rejected": -0.4773009419441223,
+      "logps/chosen": -0.5309734344482422,
+      "logps/rejected": -0.7864019870758057,
+      "loss": 0.914,
+      "odds_ratio_loss": 9.082796096801758,
+      "rewards/accuracies": 0.296875,
+      "rewards/chosen": -0.07864020019769669,
+      "rewards/margins": -0.025542862713336945,
+      "rewards/rejected": -0.05309733748435974,
+      "sft_loss": 0.005673164501786232,
+      "step": 10
+    },
+    {
+      "epoch": 0.06033596160438807,
+      "grad_norm": 6.086739375903808,
+      "learning_rate": 2.8947368421052634e-06,
+      "logits/chosen": -0.49898672103881836,
+      "logits/rejected": -0.5067213773727417,
+      "logps/chosen": -0.5432000756263733,
+      "logps/rejected": -0.728182315826416,
+      "loss": 0.8437,
+      "odds_ratio_loss": 8.347268104553223,
+      "rewards/accuracies": 0.3359375,
+      "rewards/chosen": -0.07281822711229324,
+      "rewards/margins": -0.018498217687010765,
+      "rewards/rejected": -0.05432000756263733,
+      "sft_loss": 0.00892576016485691,
+      "step": 11
+    },
+    {
+      "epoch": 0.0658210490229688,
+      "grad_norm": 2.6128540943717016,
+      "learning_rate": 3.157894736842105e-06,
+      "logits/chosen": -0.5385469198226929,
+      "logits/rejected": -0.5363799333572388,
+      "logps/chosen": -0.6628677248954773,
+      "logps/rejected": -0.7211645245552063,
+      "loss": 0.8365,
+      "odds_ratio_loss": 8.324950218200684,
+      "rewards/accuracies": 0.296875,
+      "rewards/chosen": -0.07211645692586899,
+      "rewards/margins": -0.005829682573676109,
+      "rewards/rejected": -0.06628676503896713,
+      "sft_loss": 0.003960596397519112,
+      "step": 12
+    },
+    {
+      "epoch": 0.07130613644154954,
+      "grad_norm": 3.092946841589385,
+      "learning_rate": 3.421052631578948e-06,
+      "logits/chosen": -0.5176294445991516,
+      "logits/rejected": -0.5164329409599304,
+      "logps/chosen": -0.5450438857078552,
+      "logps/rejected": -0.6731284856796265,
+      "loss": 0.7842,
+      "odds_ratio_loss": 7.793478012084961,
+      "rewards/accuracies": 0.3359375,
+      "rewards/chosen": -0.06731285154819489,
+      "rewards/margins": -0.012808457016944885,
+      "rewards/rejected": -0.054504383355379105,
+      "sft_loss": 0.004814960993826389,
+      "step": 13
+    },
+    {
+      "epoch": 0.07679122386013026,
+      "grad_norm": 2.8558270742996137,
+      "learning_rate": 3.6842105263157896e-06,
+      "logits/chosen": -0.6181472539901733,
+      "logits/rejected": -0.5952631831169128,
+      "logps/chosen": -0.6915932893753052,
+      "logps/rejected": -0.753544807434082,
+      "loss": 0.8682,
+      "odds_ratio_loss": 8.637476921081543,
+      "rewards/accuracies": 0.328125,
+      "rewards/chosen": -0.07535447925329208,
+      "rewards/margins": -0.006195154972374439,
+      "rewards/rejected": -0.06915932893753052,
+      "sft_loss": 0.0044339620508253574,
+      "step": 14
+    },
+    {
+      "epoch": 0.082276311278711,
+      "grad_norm": 3.7864729296015134,
+      "learning_rate": 3.947368421052632e-06,
+      "logits/chosen": -0.6190563440322876,
+      "logits/rejected": -0.5924232006072998,
+      "logps/chosen": -0.5566739439964294,
+      "logps/rejected": -0.7037681937217712,
+      "loss": 0.8093,
+      "odds_ratio_loss": 8.019332885742188,
+      "rewards/accuracies": 0.328125,
+      "rewards/chosen": -0.07037682831287384,
+      "rewards/margins": -0.014709431678056717,
+      "rewards/rejected": -0.05566739663481712,
+      "sft_loss": 0.007347959093749523,
+      "step": 15
+    },
+    {
+      "epoch": 0.08776139869729174,
+      "grad_norm": 3.0120281360552097,
+      "learning_rate": 4.210526315789474e-06,
+      "logits/chosen": -0.6849666833877563,
+      "logits/rejected": -0.6525503396987915,
+      "logps/chosen": -0.5696557760238647,
+      "logps/rejected": -0.7036406993865967,
+      "loss": 0.8093,
+      "odds_ratio_loss": 8.039867401123047,
+      "rewards/accuracies": 0.359375,
+      "rewards/chosen": -0.0703640729188919,
+      "rewards/margins": -0.013398496434092522,
+      "rewards/rejected": -0.05696558207273483,
+      "sft_loss": 0.005266908556222916,
+      "step": 16
+    },
+    {
+      "epoch": 0.09324648611587247,
+      "grad_norm": 2.6504291324403315,
+      "learning_rate": 4.473684210526316e-06,
+      "logits/chosen": -0.641123354434967,
+      "logits/rejected": -0.6362437605857849,
+      "logps/chosen": -0.6884087324142456,
+      "logps/rejected": -0.7263792157173157,
+      "loss": 0.8353,
+      "odds_ratio_loss": 8.31103515625,
+      "rewards/accuracies": 0.2734375,
+      "rewards/chosen": -0.07263792306184769,
+      "rewards/margins": -0.0037970547564327717,
+      "rewards/rejected": -0.0688408762216568,
+      "sft_loss": 0.004170028492808342,
+      "step": 17
+    },
+    {
+      "epoch": 0.09873157353445321,
+      "grad_norm": 2.886807767840121,
+      "learning_rate": 4.736842105263158e-06,
+      "logits/chosen": -0.6371440291404724,
+      "logits/rejected": -0.6085755825042725,
+      "logps/chosen": -0.6927422881126404,
+      "logps/rejected": -0.706704318523407,
+      "loss": 0.8013,
+      "odds_ratio_loss": 7.961912155151367,
+      "rewards/accuracies": 0.4140625,
+      "rewards/chosen": -0.07067042589187622,
+      "rewards/margins": -0.0013961929362267256,
+      "rewards/rejected": -0.06927423179149628,
+      "sft_loss": 0.005079690366983414,
+      "step": 18
+    },
+    {
+      "epoch": 0.10421666095303393,
+      "grad_norm": 5.880825411085245,
+      "learning_rate": 5e-06,
+      "logits/chosen": -0.6107761859893799,
+      "logits/rejected": -0.6409167647361755,
+      "logps/chosen": -0.628943681716919,
+      "logps/rejected": -0.6733644008636475,
+      "loss": 0.7634,
+      "odds_ratio_loss": 7.441443920135498,
+      "rewards/accuracies": 0.3359375,
+      "rewards/chosen": -0.06733644008636475,
+      "rewards/margins": -0.004442068748176098,
+      "rewards/rejected": -0.06289438158273697,
+      "sft_loss": 0.01928992196917534,
+      "step": 19
+    },
+    {
+      "epoch": 0.10970174837161467,
+      "grad_norm": 2.5047291630264965,
+      "learning_rate": 4.999535676028338e-06,
+      "logits/chosen": -0.53981614112854,
+      "logits/rejected": -0.5691806674003601,
+      "logps/chosen": -0.5706157088279724,
+      "logps/rejected": -0.6909669637680054,
+      "loss": 0.7906,
+      "odds_ratio_loss": 7.871767997741699,
+      "rewards/accuracies": 0.359375,
+      "rewards/chosen": -0.06909669935703278,
+      "rewards/margins": -0.012035122141242027,
+      "rewards/rejected": -0.0570615753531456,
+      "sft_loss": 0.003470680210739374,
+      "step": 20
+    },
+    {
+      "epoch": 0.11518683579019541,
+      "grad_norm": 2.731427921236176,
+      "learning_rate": 4.998142876590749e-06,
+      "logits/chosen": -0.6109368801116943,
+      "logits/rejected": -0.5822383165359497,
+      "logps/chosen": -0.5740761756896973,
+      "logps/rejected": -0.7196378111839294,
+      "loss": 0.8229,
+      "odds_ratio_loss": 8.185101509094238,
+      "rewards/accuracies": 0.3203125,
+      "rewards/chosen": -0.07196377962827682,
+      "rewards/margins": -0.014556167647242546,
+      "rewards/rejected": -0.05740761756896973,
+      "sft_loss": 0.004432335030287504,
+      "step": 21
+    },
+    {
+      "epoch": 0.12067192320877614,
+      "grad_norm": 2.6457518569851914,
+      "learning_rate": 4.9958221190553705e-06,
+      "logits/chosen": -0.6459155082702637,
+      "logits/rejected": -0.6224872469902039,
+      "logps/chosen": -0.9075093269348145,
+      "logps/rejected": -0.6858919858932495,
+      "loss": 0.7814,
+      "odds_ratio_loss": 7.773059844970703,
+      "rewards/accuracies": 0.3359375,
+      "rewards/chosen": -0.06858920305967331,
+      "rewards/margins": 0.02216172218322754,
+      "rewards/rejected": -0.09075092524290085,
+      "sft_loss": 0.004068214446306229,
+      "step": 22
+    },
+    {
+      "epoch": 0.12615701062735687,
+      "grad_norm": 2.3568903077737398,
+      "learning_rate": 4.992574265488883e-06,
+      "logits/chosen": -0.596120297908783,
+      "logits/rejected": -0.5599789619445801,
+      "logps/chosen": -0.6051114201545715,
+      "logps/rejected": -0.6642252206802368,
+      "loss": 0.7587,
+      "odds_ratio_loss": 7.555657386779785,
+      "rewards/accuracies": 0.3203125,
+      "rewards/chosen": -0.06642251461744308,
+      "rewards/margins": -0.0059113758616149426,
+      "rewards/rejected": -0.06051114201545715,
+      "sft_loss": 0.0031609206926077604,
+      "step": 23
+    },
+    {
+      "epoch": 0.1316420980459376,
+      "grad_norm": 2.4794879192215245,
+      "learning_rate": 4.988400522336304e-06,
+      "logits/chosen": -0.6102786064147949,
+      "logits/rejected": -0.5751500129699707,
+      "logps/chosen": -0.6058512926101685,
+      "logps/rejected": -0.6483108997344971,
+      "loss": 0.7446,
+      "odds_ratio_loss": 7.4141364097595215,
+      "rewards/accuracies": 0.40625,
+      "rewards/chosen": -0.06483108550310135,
+      "rewards/margins": -0.004245956428349018,
+      "rewards/rejected": -0.060585130006074905,
+      "sft_loss": 0.003168251132592559,
+      "step": 24
+    },
+    {
+      "epoch": 0.13712718546451835,
+      "grad_norm": 2.6638728856099076,
+      "learning_rate": 4.9833024399728295e-06,
+      "logits/chosen": -0.6613443493843079,
+      "logits/rejected": -0.6328415870666504,
+      "logps/chosen": -0.6213655471801758,
+      "logps/rejected": -0.6468154191970825,
+      "loss": 0.7378,
+      "odds_ratio_loss": 7.336475372314453,
+      "rewards/accuracies": 0.3984375,
+      "rewards/chosen": -0.06468154489994049,
+      "rewards/margins": -0.0025449953973293304,
+      "rewards/rejected": -0.06213654577732086,
+      "sft_loss": 0.004186041187494993,
+      "step": 25
+    },
+    {
+      "epoch": 0.14261227288309908,
+      "grad_norm": 2.487669137174005,
+      "learning_rate": 4.9772819121279395e-06,
+      "logits/chosen": -0.6384425759315491,
+      "logits/rejected": -0.6189238429069519,
+      "logps/chosen": -0.5178334712982178,
+      "logps/rejected": -0.6201947927474976,
+      "loss": 0.7189,
+      "odds_ratio_loss": 7.1527557373046875,
+      "rewards/accuracies": 0.421875,
+      "rewards/chosen": -0.0620194748044014,
+      "rewards/margins": -0.010236131027340889,
+      "rewards/rejected": -0.05178334563970566,
+      "sft_loss": 0.00362430140376091,
+      "step": 26
+    },
+    {
+      "epoch": 0.1480973603016798,
+      "grad_norm": 2.3790278282662314,
+      "learning_rate": 4.970341175181957e-06,
+      "logits/chosen": -0.6167687177658081,
+      "logits/rejected": -0.6090523600578308,
+      "logps/chosen": -0.5448204278945923,
+      "logps/rejected": -0.660010576248169,
+      "loss": 0.7598,
+      "odds_ratio_loss": 7.5589399337768555,
+      "rewards/accuracies": 0.3203125,
+      "rewards/chosen": -0.06600106507539749,
+      "rewards/margins": -0.011519018560647964,
+      "rewards/rejected": -0.05448204651474953,
+      "sft_loss": 0.00385806686244905,
+      "step": 27
+    },
+    {
+      "epoch": 0.15358244772026053,
+      "grad_norm": 2.6617242551359275,
+      "learning_rate": 4.9624828073353144e-06,
+      "logits/chosen": -0.6628867387771606,
+      "logits/rejected": -0.642227053642273,
+      "logps/chosen": -0.5881469249725342,
+      "logps/rejected": -0.6607677340507507,
+      "loss": 0.7543,
+      "odds_ratio_loss": 7.503334045410156,
+      "rewards/accuracies": 0.3828125,
+      "rewards/chosen": -0.06607677042484283,
+      "rewards/margins": -0.007262084167450666,
+      "rewards/rejected": -0.05881468579173088,
+      "sft_loss": 0.003945589065551758,
+      "step": 28
+    },
+    {
+      "epoch": 0.15906753513884128,
+      "grad_norm": 5.959951004497131,
+      "learning_rate": 4.95370972765087e-06,
+      "logits/chosen": -0.6791335344314575,
+      "logits/rejected": -0.6436203122138977,
+      "logps/chosen": -0.6152352094650269,
+      "logps/rejected": -0.7523278594017029,
+      "loss": 0.853,
+      "odds_ratio_loss": 8.355252265930176,
+      "rewards/accuracies": 0.28125,
+      "rewards/chosen": -0.07523278146982193,
+      "rewards/margins": -0.013709258288145065,
+      "rewards/rejected": -0.06152352690696716,
+      "sft_loss": 0.017459843307733536,
+      "step": 29
+    },
+    {
+      "epoch": 0.164552622557422,
+      "grad_norm": 2.3211607809375328,
+      "learning_rate": 4.944025194969586e-06,
+      "logits/chosen": -0.6864458322525024,
+      "logits/rejected": -0.6491591930389404,
+      "logps/chosen": -0.591070294380188,
+      "logps/rejected": -0.6974666118621826,
+      "loss": 0.7936,
+      "odds_ratio_loss": 7.89208984375,
+      "rewards/accuracies": 0.3984375,
+      "rewards/chosen": -0.06974666565656662,
+      "rewards/margins": -0.010639630258083344,
+      "rewards/rejected": -0.05910703167319298,
+      "sft_loss": 0.004418414086103439,
+      "step": 30
+    },
+    {
+      "epoch": 0.17003770997600273,
+      "grad_norm": 2.4315181303090307,
+      "learning_rate": 4.933432806700004e-06,
+      "logits/chosen": -0.6587315797805786,
+      "logits/rejected": -0.6222696304321289,
+      "logps/chosen": -0.6303814053535461,
+      "logps/rejected": -0.7028146982192993,
+      "loss": 0.8067,
+      "odds_ratio_loss": 8.026537895202637,
+      "rewards/accuracies": 0.375,
+      "rewards/chosen": -0.07028146833181381,
+      "rewards/margins": -0.007243326865136623,
+      "rewards/rejected": -0.06303813308477402,
+      "sft_loss": 0.004071646369993687,
+      "step": 31
+    },
+    {
+      "epoch": 0.17552279739458349,
+      "grad_norm": 2.210136151504232,
+      "learning_rate": 4.921936497481956e-06,
+      "logits/chosen": -0.7069422006607056,
+      "logits/rejected": -0.6679366230964661,
+      "logps/chosen": -0.5178747177124023,
+      "logps/rejected": -0.6546376347541809,
+      "loss": 0.7551,
+      "odds_ratio_loss": 7.520661354064941,
+      "rewards/accuracies": 0.3515625,
+      "rewards/chosen": -0.06546376645565033,
+      "rewards/margins": -0.013676293194293976,
+      "rewards/rejected": -0.05178747698664665,
+      "sft_loss": 0.003003381658345461,
+      "step": 32
+    },
+    {
+      "epoch": 0.1810078848131642,
+      "grad_norm": 3.4053506226819574,
+      "learning_rate": 4.909540537725007e-06,
+      "logits/chosen": -0.6776018142700195,
+      "logits/rejected": -0.6548346877098083,
+      "logps/chosen": -0.6834210157394409,
+      "logps/rejected": -0.6428653001785278,
+      "loss": 0.7378,
+      "odds_ratio_loss": 7.308858871459961,
+      "rewards/accuracies": 0.421875,
+      "rewards/chosen": -0.06428653001785278,
+      "rewards/margins": 0.004055576398968697,
+      "rewards/rejected": -0.06834210455417633,
+      "sft_loss": 0.006872784812003374,
+      "step": 33
+    },
+    {
+      "epoch": 0.18649297223174494,
+      "grad_norm": 2.539813335312958,
+      "learning_rate": 4.8962495320221714e-06,
+      "logits/chosen": -0.7110268473625183,
+      "logits/rejected": -0.6500384211540222,
+      "logps/chosen": -0.5525631308555603,
+      "logps/rejected": -0.673240065574646,
+      "loss": 0.7743,
+      "odds_ratio_loss": 7.700582027435303,
+      "rewards/accuracies": 0.3359375,
+      "rewards/chosen": -0.06732401251792908,
+      "rewards/margins": -0.012067697942256927,
+      "rewards/rejected": -0.05525631457567215,
+      "sft_loss": 0.0042497809045016766,
+      "step": 34
+    },
+    {
+      "epoch": 0.1919780596503257,
+      "grad_norm": 2.655970243707059,
+      "learning_rate": 4.8820684174394935e-06,
+      "logits/chosen": -0.7138683199882507,
+      "logits/rejected": -0.6818023920059204,
+      "logps/chosen": -0.5597530603408813,
+      "logps/rejected": -0.6535578966140747,
+      "loss": 0.7491,
+      "odds_ratio_loss": 7.4469194412231445,
+      "rewards/accuracies": 0.375,
+      "rewards/chosen": -0.06535579264163971,
+      "rewards/margins": -0.009380483068525791,
+      "rewards/rejected": -0.0559752993285656,
+      "sft_loss": 0.004387051332741976,
+      "step": 35
+    },
+    {
+      "epoch": 0.19746314706890641,
+      "grad_norm": 2.5709022001566275,
+      "learning_rate": 4.867002461682129e-06,
+      "logits/chosen": -0.6977266073226929,
+      "logits/rejected": -0.6760712265968323,
+      "logps/chosen": -0.5894060134887695,
+      "logps/rejected": -0.7010530829429626,
+      "loss": 0.8056,
+      "odds_ratio_loss": 8.00703239440918,
+      "rewards/accuracies": 0.3828125,
+      "rewards/chosen": -0.07010531425476074,
+      "rewards/margins": -0.011164708994328976,
+      "rewards/rejected": -0.058940596878528595,
+      "sft_loss": 0.004914172925055027,
+      "step": 36
+    },
+    {
+      "epoch": 0.20294823448748714,
+      "grad_norm": 2.3705551951599895,
+      "learning_rate": 4.851057261137608e-06,
+      "logits/chosen": -0.69024658203125,
+      "logits/rejected": -0.6706061363220215,
+      "logps/chosen": -0.5469837784767151,
+      "logps/rejected": -0.6902478933334351,
+      "loss": 0.7905,
+      "odds_ratio_loss": 7.859821319580078,
+      "rewards/accuracies": 0.34375,
+      "rewards/chosen": -0.06902480125427246,
+      "rewards/margins": -0.014326417818665504,
+      "rewards/rejected": -0.05469837784767151,
+      "sft_loss": 0.0044726720079779625,
+      "step": 37
+    },
+    {
+      "epoch": 0.20843332190606786,
+      "grad_norm": 5.234021982751859,
+      "learning_rate": 4.8342387387970105e-06,
+      "logits/chosen": -0.6684221625328064,
+      "logits/rejected": -0.6224137544631958,
+      "logps/chosen": -0.6020541191101074,
+      "logps/rejected": -0.6657444834709167,
+      "loss": 0.7536,
+      "odds_ratio_loss": 7.442554473876953,
+      "rewards/accuracies": 0.4140625,
+      "rewards/chosen": -0.06657445430755615,
+      "rewards/margins": -0.0063690319657325745,
+      "rewards/rejected": -0.06020541861653328,
+      "sft_loss": 0.009305169805884361,
+      "step": 38
+    },
+    {
+      "epoch": 0.21391840932464862,
+      "grad_norm": 2.0334062957234833,
+      "learning_rate": 4.816553142054806e-06,
+      "logits/chosen": -0.6587538719177246,
+      "logits/rejected": -0.6190416216850281,
+      "logps/chosen": -0.5317018032073975,
+      "logps/rejected": -0.6559545993804932,
+      "loss": 0.7589,
+      "odds_ratio_loss": 7.557468414306641,
+      "rewards/accuracies": 0.328125,
+      "rewards/chosen": -0.06559545546770096,
+      "rewards/margins": -0.012425270862877369,
+      "rewards/rejected": -0.053170185536146164,
+      "sft_loss": 0.003142669564113021,
+      "step": 39
+    },
+    {
+      "epoch": 0.21940349674322934,
+      "grad_norm": 2.615502723756528,
+      "learning_rate": 4.798007040388212e-06,
+      "logits/chosen": -0.6537474393844604,
+      "logits/rejected": -0.6558958888053894,
+      "logps/chosen": -0.5594260692596436,
+      "logps/rejected": -0.7137230634689331,
+      "loss": 0.8176,
+      "odds_ratio_loss": 8.131279945373535,
+      "rewards/accuracies": 0.296875,
+      "rewards/chosen": -0.07137230783700943,
+      "rewards/margins": -0.015429697930812836,
+      "rewards/rejected": -0.055942609906196594,
+      "sft_loss": 0.004487923812121153,
+      "step": 40
+    },
+    {
+      "epoch": 0.22488858416181007,
+      "grad_norm": 3.147981874003909,
+      "learning_rate": 4.778607322916896e-06,
+      "logits/chosen": -0.7805840969085693,
+      "logits/rejected": -0.7332777976989746,
+      "logps/chosen": -0.5440269708633423,
+      "logps/rejected": -0.6570194959640503,
+      "loss": 0.755,
+      "odds_ratio_loss": 7.461404800415039,
+      "rewards/accuracies": 0.34375,
+      "rewards/chosen": -0.06570195406675339,
+      "rewards/margins": -0.011299250647425652,
+      "rewards/rejected": -0.05440270155668259,
+      "sft_loss": 0.008839858695864677,
+      "step": 41
+    },
+    {
+      "epoch": 0.23037367158039082,
+      "grad_norm": 2.4132096188646868,
+      "learning_rate": 4.7583611958439514e-06,
+      "logits/chosen": -0.6508989334106445,
+      "logits/rejected": -0.6227612495422363,
+      "logps/chosen": -0.6552981734275818,
+      "logps/rejected": -0.7332005500793457,
+      "loss": 0.8445,
+      "odds_ratio_loss": 8.40545654296875,
+      "rewards/accuracies": 0.2734375,
+      "rewards/chosen": -0.07332006096839905,
+      "rewards/margins": -0.007790243253111839,
+      "rewards/rejected": -0.06552981585264206,
+      "sft_loss": 0.0039908913895487785,
+      "step": 42
+    },
+    {
+      "epoch": 0.23585875899897155,
+      "grad_norm": 2.029276338522748,
+      "learning_rate": 4.7372761797790836e-06,
+      "logits/chosen": -0.6611707210540771,
+      "logits/rejected": -0.6425488591194153,
+      "logps/chosen": -0.5845383405685425,
+      "logps/rejected": -0.6395372748374939,
+      "loss": 0.7307,
+      "odds_ratio_loss": 7.273993968963623,
+      "rewards/accuracies": 0.3828125,
+      "rewards/chosen": -0.06395373493432999,
+      "rewards/margins": -0.005499903112649918,
+      "rewards/rejected": -0.05845382809638977,
+      "sft_loss": 0.003296809270977974,
+      "step": 43
+    },
+    {
+      "epoch": 0.24134384641755227,
+      "grad_norm": 2.334850850202144,
+      "learning_rate": 4.715360106945015e-06,
+      "logits/chosen": -0.6538242101669312,
+      "logits/rejected": -0.6651148796081543,
+      "logps/chosen": -0.5922138690948486,
+      "logps/rejected": -0.6789554357528687,
+      "loss": 0.7687,
+      "odds_ratio_loss": 7.646347999572754,
+      "rewards/accuracies": 0.359375,
+      "rewards/chosen": -0.0678955465555191,
+      "rewards/margins": -0.008674154058098793,
+      "rewards/rejected": -0.059221383184194565,
+      "sft_loss": 0.004072089679539204,
+      "step": 44
+    },
+    {
+      "epoch": 0.24682893383613302,
+      "grad_norm": 3.278246254509892,
+      "learning_rate": 4.6926211182681295e-06,
+      "logits/chosen": -0.7525214552879333,
+      "logits/rejected": -0.7238048315048218,
+      "logps/chosen": -0.6908121109008789,
+      "logps/rejected": -0.6656503677368164,
+      "loss": 0.7586,
+      "odds_ratio_loss": 7.535221099853516,
+      "rewards/accuracies": 0.3828125,
+      "rewards/chosen": -0.06656503677368164,
+      "rewards/margins": 0.0025161777157336473,
+      "rewards/rejected": -0.06908121705055237,
+      "sft_loss": 0.005099683068692684,
+      "step": 45
+    },
+    {
+      "epoch": 0.25231402125471375,
+      "grad_norm": 2.1664816320305897,
+      "learning_rate": 4.669067660354456e-06,
+      "logits/chosen": -0.6813299059867859,
+      "logits/rejected": -0.6507269144058228,
+      "logps/chosen": -0.5852293372154236,
+      "logps/rejected": -0.6381067633628845,
+      "loss": 0.726,
+      "odds_ratio_loss": 7.226728439331055,
+      "rewards/accuracies": 0.3515625,
+      "rewards/chosen": -0.06381067633628845,
+      "rewards/margins": -0.005287742242217064,
+      "rewards/rejected": -0.05852293595671654,
+      "sft_loss": 0.003340219147503376,
+      "step": 46
+    },
+    {
+      "epoch": 0.2577991086732945,
+      "grad_norm": 2.3755468836145535,
+      "learning_rate": 4.644708482352093e-06,
+      "logits/chosen": -0.7196726202964783,
+      "logits/rejected": -0.671117901802063,
+      "logps/chosen": -0.496429979801178,
+      "logps/rejected": -0.6285478472709656,
+      "loss": 0.7368,
+      "odds_ratio_loss": 7.332725524902344,
+      "rewards/accuracies": 0.3203125,
+      "rewards/chosen": -0.06285478919744492,
+      "rewards/margins": -0.013211790472269058,
+      "rewards/rejected": -0.04964299499988556,
+      "sft_loss": 0.003541831858456135,
+      "step": 47
+    },
+    {
+      "epoch": 0.2632841960918752,
+      "grad_norm": 1.882676786275039,
+      "learning_rate": 4.619552632701263e-06,
+      "logits/chosen": -0.7029179334640503,
+      "logits/rejected": -0.6741898059844971,
+      "logps/chosen": -0.5508084297180176,
+      "logps/rejected": -0.6012480854988098,
+      "loss": 0.6905,
+      "odds_ratio_loss": 6.879641532897949,
+      "rewards/accuracies": 0.3828125,
+      "rewards/chosen": -0.06012480705976486,
+      "rewards/margins": -0.005043962970376015,
+      "rewards/rejected": -0.055080845952034,
+      "sft_loss": 0.0025444331113249063,
+      "step": 48
+    },
+    {
+      "epoch": 0.2687692835104559,
+      "grad_norm": 2.3230672316134244,
+      "learning_rate": 4.5936094557731815e-06,
+      "logits/chosen": -0.6754356622695923,
+      "logits/rejected": -0.6543477177619934,
+      "logps/chosen": -0.5631517171859741,
+      "logps/rejected": -0.6714473366737366,
+      "loss": 0.7666,
+      "odds_ratio_loss": 7.623527526855469,
+      "rewards/accuracies": 0.328125,
+      "rewards/chosen": -0.0671447217464447,
+      "rewards/margins": -0.010829559527337551,
+      "rewards/rejected": -0.05631516873836517,
+      "sft_loss": 0.004294781945645809,
+      "step": 49
+    },
+    {
+      "epoch": 0.2742543709290367,
+      "grad_norm": 2.3609733896189353,
+      "learning_rate": 4.566888588399007e-06,
+      "logits/chosen": -0.6326004266738892,
+      "logits/rejected": -0.6783662438392639,
+      "logps/chosen": -0.6179240345954895,
+      "logps/rejected": -0.64500892162323,
+      "loss": 0.7366,
+      "odds_ratio_loss": 7.319670677185059,
+      "rewards/accuracies": 0.4140625,
+      "rewards/chosen": -0.06450089812278748,
+      "rewards/margins": -0.002708489540964365,
+      "rewards/rejected": -0.06179240718483925,
+      "sft_loss": 0.0045843422412872314,
+      "step": 50
+    },
+    {
+      "epoch": 0.27973945834761743,
+      "grad_norm": 2.150795076150455,
+      "learning_rate": 4.539399956290152e-06,
+      "logits/chosen": -0.6697653532028198,
+      "logits/rejected": -0.6334677934646606,
+      "logps/chosen": -0.5712593197822571,
+      "logps/rejected": -0.5926052927970886,
+      "loss": 0.6852,
+      "odds_ratio_loss": 6.818970680236816,
+      "rewards/accuracies": 0.3828125,
+      "rewards/chosen": -0.0592605359852314,
+      "rewards/margins": -0.002134602749720216,
+      "rewards/rejected": -0.05712592974305153,
+      "sft_loss": 0.003330723848193884,
+      "step": 51
+    },
+    {
+      "epoch": 0.28522454576619816,
+      "grad_norm": 2.2865734329168563,
+      "learning_rate": 4.511153770351288e-06,
+      "logits/chosen": -0.7038233280181885,
+      "logits/rejected": -0.6797916889190674,
+      "logps/chosen": -0.626400351524353,
+      "logps/rejected": -0.6849571466445923,
+      "loss": 0.7765,
+      "odds_ratio_loss": 7.725215435028076,
+      "rewards/accuracies": 0.328125,
+      "rewards/chosen": -0.06849571317434311,
+      "rewards/margins": -0.0058556715957820415,
+      "rewards/rejected": -0.06264004111289978,
+      "sft_loss": 0.00396262900903821,
+      "step": 52
+    },
+    {
+      "epoch": 0.2907096331847789,
+      "grad_norm": 2.3920025552439967,
+      "learning_rate": 4.482160522887404e-06,
+      "logits/chosen": -0.6559309363365173,
+      "logits/rejected": -0.6758289337158203,
+      "logps/chosen": -0.6616750955581665,
+      "logps/rejected": -0.6394928693771362,
+      "loss": 0.7281,
+      "odds_ratio_loss": 7.238113880157471,
+      "rewards/accuracies": 0.390625,
+      "rewards/chosen": -0.06394927203655243,
+      "rewards/margins": 0.0022182257380336523,
+      "rewards/rejected": -0.06616750359535217,
+      "sft_loss": 0.004315241239964962,
+      "step": 53
+    },
+    {
+      "epoch": 0.2961947206033596,
+      "grad_norm": 2.5117956914717032,
+      "learning_rate": 4.452430983706351e-06,
+      "logits/chosen": -0.7268598079681396,
+      "logits/rejected": -0.7092704772949219,
+      "logps/chosen": -0.549469530582428,
+      "logps/rejected": -0.6699557304382324,
+      "loss": 0.7657,
+      "odds_ratio_loss": 7.608267784118652,
+      "rewards/accuracies": 0.3671875,
+      "rewards/chosen": -0.06699557602405548,
+      "rewards/margins": -0.012048620730638504,
+      "rewards/rejected": -0.054946959018707275,
+      "sft_loss": 0.004888926167041063,
+      "step": 54
+    },
+    {
+      "epoch": 0.30167980802194033,
+      "grad_norm": 6.1164968534949375,
+      "learning_rate": 4.421976196118297e-06,
+      "logits/chosen": -0.6996357440948486,
+      "logits/rejected": -0.6125066876411438,
+      "logps/chosen": -0.5666108131408691,
+      "logps/rejected": -0.7606990933418274,
+      "loss": 0.8647,
+      "odds_ratio_loss": 7.164684295654297,
+      "rewards/accuracies": 0.3984375,
+      "rewards/chosen": -0.0760699063539505,
+      "rewards/margins": -0.019408833235502243,
+      "rewards/rejected": -0.056661076843738556,
+      "sft_loss": 0.14825321733951569,
+      "step": 55
+    },
+    {
+      "epoch": 0.30716489544052106,
+      "grad_norm": 2.3449325969971553,
+      "learning_rate": 4.390807472833585e-06,
+      "logits/chosen": -0.7196471095085144,
+      "logits/rejected": -0.6497766375541687,
+      "logps/chosen": -0.5705811977386475,
+      "logps/rejected": -0.6152413487434387,
+      "loss": 0.7056,
+      "odds_ratio_loss": 7.016576766967773,
+      "rewards/accuracies": 0.4140625,
+      "rewards/chosen": -0.061524130403995514,
+      "rewards/margins": -0.004466014914214611,
+      "rewards/rejected": -0.05705811828374863,
+      "sft_loss": 0.003966608550399542,
+      "step": 56
+    },
+    {
+      "epoch": 0.31264998285910184,
+      "grad_norm": 1.9758244727864838,
+      "learning_rate": 4.358936391760524e-06,
+      "logits/chosen": -0.6982048749923706,
+      "logits/rejected": -0.6489894986152649,
+      "logps/chosen": -0.7246107459068298,
+      "logps/rejected": -0.6382527947425842,
+      "loss": 0.7298,
+      "odds_ratio_loss": 7.263551235198975,
+      "rewards/accuracies": 0.3984375,
+      "rewards/chosen": -0.06382527947425842,
+      "rewards/margins": 0.008635802194476128,
+      "rewards/rejected": -0.0724610835313797,
+      "sft_loss": 0.0034022387117147446,
+      "step": 57
+    },
+    {
+      "epoch": 0.31813507027768256,
+      "grad_norm": 2.174135911231623,
+      "learning_rate": 4.32637479170467e-06,
+      "logits/chosen": -0.7073652148246765,
+      "logits/rejected": -0.6690998077392578,
+      "logps/chosen": -0.6008009314537048,
+      "logps/rejected": -0.6260058879852295,
+      "loss": 0.7136,
+      "odds_ratio_loss": 7.0961079597473145,
+      "rewards/accuracies": 0.4375,
+      "rewards/chosen": -0.06260059028863907,
+      "rewards/margins": -0.0025204988196492195,
+      "rewards/rejected": -0.060080088675022125,
+      "sft_loss": 0.00403111707419157,
+      "step": 58
+    },
+    {
+      "epoch": 0.3236201576962633,
+      "grad_norm": 2.168405994153748,
+      "learning_rate": 4.293134767971193e-06,
+      "logits/chosen": -0.7020463347434998,
+      "logits/rejected": -0.6942731142044067,
+      "logps/chosen": -0.5773230791091919,
+      "logps/rejected": -0.6443250179290771,
+      "loss": 0.7339,
+      "odds_ratio_loss": 7.302745342254639,
+      "rewards/accuracies": 0.375,
+      "rewards/chosen": -0.06443249434232712,
+      "rewards/margins": -0.006700189784169197,
+      "rewards/rejected": -0.05773231014609337,
+      "sft_loss": 0.003663026262074709,
+      "step": 59
+    },
+    {
+      "epoch": 0.329105245114844,
+      "grad_norm": 2.185211880662804,
+      "learning_rate": 4.259228667871963e-06,
+      "logits/chosen": -0.6689984202384949,
+      "logits/rejected": -0.675392746925354,
+      "logps/chosen": -0.6228610277175903,
+      "logps/rejected": -0.6264554262161255,
+      "loss": 0.7157,
+      "odds_ratio_loss": 7.120779991149902,
+      "rewards/accuracies": 0.40625,
+      "rewards/chosen": -0.06264554709196091,
+      "rewards/margins": -0.0003594460431486368,
+      "rewards/rejected": -0.062286097556352615,
+      "sft_loss": 0.0035854382440447807,
+      "step": 60
+    },
+    {
+      "epoch": 0.33459033253342474,
+      "grad_norm": 2.57070026430243,
+      "learning_rate": 4.22466908613903e-06,
+      "logits/chosen": -0.702937126159668,
+      "logits/rejected": -0.7012047171592712,
+      "logps/chosen": -0.7283627986907959,
+      "logps/rejected": -0.6880404949188232,
+      "loss": 0.7811,
+      "odds_ratio_loss": 7.746993064880371,
+      "rewards/accuracies": 0.3671875,
+      "rewards/chosen": -0.0688040480017662,
+      "rewards/margins": 0.0040322281420230865,
+      "rewards/rejected": -0.07283627241849899,
+      "sft_loss": 0.006447223015129566,
+      "step": 61
+    },
+    {
+      "epoch": 0.34007541995200546,
+      "grad_norm": 4.244463596912575,
+      "learning_rate": 4.189468860246192e-06,
+      "logits/chosen": -0.6791540384292603,
+      "logits/rejected": -0.6635554432868958,
+      "logps/chosen": -0.6107332706451416,
+      "logps/rejected": -0.6247188448905945,
+      "loss": 0.7142,
+      "odds_ratio_loss": 7.000344753265381,
+      "rewards/accuracies": 0.3984375,
+      "rewards/chosen": -0.06247188523411751,
+      "rewards/margins": -0.001398557680658996,
+      "rewards/rejected": -0.06107332557439804,
+      "sft_loss": 0.014190858229994774,
+      "step": 62
+    },
+    {
+      "epoch": 0.34556050737058625,
+      "grad_norm": 2.025728542662067,
+      "learning_rate": 4.153641065640402e-06,
+      "logits/chosen": -0.693527340888977,
+      "logits/rejected": -0.6523553133010864,
+      "logps/chosen": -0.5719731450080872,
+      "logps/rejected": -0.6185833811759949,
+      "loss": 0.7056,
+      "odds_ratio_loss": 7.020816802978516,
+      "rewards/accuracies": 0.375,
+      "rewards/chosen": -0.06185833364725113,
+      "rewards/margins": -0.004661021754145622,
+      "rewards/rejected": -0.057197313755750656,
+      "sft_loss": 0.003522280603647232,
+      "step": 63
+    },
+    {
+      "epoch": 0.35104559478916697,
+      "grad_norm": 2.295177803143149,
+      "learning_rate": 4.1171990108847705e-06,
+      "logits/chosen": -0.6868746280670166,
+      "logits/rejected": -0.6699408888816833,
+      "logps/chosen": -0.6467013359069824,
+      "logps/rejected": -0.6266091465950012,
+      "loss": 0.719,
+      "odds_ratio_loss": 7.1493024826049805,
+      "rewards/accuracies": 0.3984375,
+      "rewards/chosen": -0.06266091763973236,
+      "rewards/margins": 0.002009219955652952,
+      "rewards/rejected": -0.0646701380610466,
+      "sft_loss": 0.004113649483770132,
+      "step": 64
+    },
+    {
+      "epoch": 0.3565306822077477,
+      "grad_norm": 2.2636310766420693,
+      "learning_rate": 4.080156232714976e-06,
+      "logits/chosen": -0.6988135576248169,
+      "logits/rejected": -0.6200304627418518,
+      "logps/chosen": -0.560631513595581,
+      "logps/rejected": -0.6326756477355957,
+      "loss": 0.7288,
+      "odds_ratio_loss": 7.247946262359619,
+      "rewards/accuracies": 0.3828125,
+      "rewards/chosen": -0.06326755881309509,
+      "rewards/margins": -0.007204408757388592,
+      "rewards/rejected": -0.056063152849674225,
+      "sft_loss": 0.004002667032182217,
+      "step": 65
+    },
+    {
+      "epoch": 0.3620157696263284,
+      "grad_norm": 2.3726953806807543,
+      "learning_rate": 4.0425264910109245e-06,
+      "logits/chosen": -0.6342322826385498,
+      "logits/rejected": -0.6143465042114258,
+      "logps/chosen": -0.5673171877861023,
+      "logps/rejected": -0.6150503158569336,
+      "loss": 0.704,
+      "odds_ratio_loss": 6.996849060058594,
+      "rewards/accuracies": 0.3515625,
+      "rewards/chosen": -0.0615050233900547,
+      "rewards/margins": -0.004773305729031563,
+      "rewards/rejected": -0.05673171579837799,
+      "sft_loss": 0.004326392896473408,
+      "step": 66
+    },
+    {
+      "epoch": 0.36750085704490915,
+      "grad_norm": 2.3264926175994343,
+      "learning_rate": 4.004323763685511e-06,
+      "logits/chosen": -0.6961945295333862,
+      "logits/rejected": -0.6595050096511841,
+      "logps/chosen": -0.7507973909378052,
+      "logps/rejected": -0.6752977967262268,
+      "loss": 0.7631,
+      "odds_ratio_loss": 7.585504531860352,
+      "rewards/accuracies": 0.4140625,
+      "rewards/chosen": -0.06752977520227432,
+      "rewards/margins": 0.0075499615631997585,
+      "rewards/rejected": -0.07507973909378052,
+      "sft_loss": 0.004573096055537462,
+      "step": 67
+    },
+    {
+      "epoch": 0.37298594446348987,
+      "grad_norm": 2.1356607945545245,
+      "learning_rate": 3.965562241492401e-06,
+      "logits/chosen": -0.6466645002365112,
+      "logits/rejected": -0.5876697301864624,
+      "logps/chosen": -0.6032683253288269,
+      "logps/rejected": -0.6798368692398071,
+      "loss": 0.7748,
+      "odds_ratio_loss": 7.713741779327393,
+      "rewards/accuracies": 0.3359375,
+      "rewards/chosen": -0.06798368692398071,
+      "rewards/margins": -0.007656855508685112,
+      "rewards/rejected": -0.06032683700323105,
+      "sft_loss": 0.0033814627677202225,
+      "step": 68
+    },
+    {
+      "epoch": 0.3784710318820706,
+      "grad_norm": 2.2504419735299632,
+      "learning_rate": 3.92625632275474e-06,
+      "logits/chosen": -0.6422359943389893,
+      "logits/rejected": -0.635747492313385,
+      "logps/chosen": -0.6667078137397766,
+      "logps/rejected": -0.6623092889785767,
+      "loss": 0.7479,
+      "odds_ratio_loss": 7.437192916870117,
+      "rewards/accuracies": 0.3828125,
+      "rewards/chosen": -0.06623092293739319,
+      "rewards/margins": 0.00043986161472275853,
+      "rewards/rejected": -0.06667079031467438,
+      "sft_loss": 0.004172264598309994,
+      "step": 69
+    },
+    {
+      "epoch": 0.3839561193006514,
+      "grad_norm": 2.305721199557074,
+      "learning_rate": 3.886420608016767e-06,
+      "logits/chosen": -0.6914317607879639,
+      "logits/rejected": -0.6497669219970703,
+      "logps/chosen": -0.6576675176620483,
+      "logps/rejected": -0.6327704191207886,
+      "loss": 0.72,
+      "odds_ratio_loss": 7.161937236785889,
+      "rewards/accuracies": 0.421875,
+      "rewards/chosen": -0.06327703595161438,
+      "rewards/margins": 0.002489713719114661,
+      "rewards/rejected": -0.06576675176620483,
+      "sft_loss": 0.003831625683233142,
+      "step": 70
+    },
+    {
+      "epoch": 0.3894412067192321,
+      "grad_norm": 2.3551007707163962,
+      "learning_rate": 3.846069894620306e-06,
+      "logits/chosen": -0.6762869358062744,
+      "logits/rejected": -0.689199686050415,
+      "logps/chosen": -0.5313611030578613,
+      "logps/rejected": -0.6086665391921997,
+      "loss": 0.6972,
+      "odds_ratio_loss": 6.926117420196533,
+      "rewards/accuracies": 0.3828125,
+      "rewards/chosen": -0.06086665764451027,
+      "rewards/margins": -0.007730549667030573,
+      "rewards/rejected": -0.05313611030578613,
+      "sft_loss": 0.0045737153850495815,
+      "step": 71
+    },
+    {
+      "epoch": 0.39492629413781283,
+      "grad_norm": 2.556752307102043,
+      "learning_rate": 3.80521917120816e-06,
+      "logits/chosen": -0.6555116772651672,
+      "logits/rejected": -0.6547709107398987,
+      "logps/chosen": -0.624289333820343,
+      "logps/rejected": -0.6435787081718445,
+      "loss": 0.7314,
+      "odds_ratio_loss": 7.269766330718994,
+      "rewards/accuracies": 0.3828125,
+      "rewards/chosen": -0.06435786932706833,
+      "rewards/margins": -0.001928933197632432,
+      "rewards/rejected": -0.06242894381284714,
+      "sft_loss": 0.00443875789642334,
+      "step": 72
+    },
+    {
+      "epoch": 0.40041138155639355,
+      "grad_norm": 2.62244580300382,
+      "learning_rate": 3.7638836121564414e-06,
+      "logits/chosen": -0.633210301399231,
+      "logits/rejected": -0.6068264245986938,
+      "logps/chosen": -0.6534138917922974,
+      "logps/rejected": -0.6456267833709717,
+      "loss": 0.7342,
+      "odds_ratio_loss": 7.288712501525879,
+      "rewards/accuracies": 0.390625,
+      "rewards/chosen": -0.06456267833709717,
+      "rewards/margins": 0.0007787136128172278,
+      "rewards/rejected": -0.06534139811992645,
+      "sft_loss": 0.005336450412869453,
+      "step": 73
+    },
+    {
+      "epoch": 0.4058964689749743,
+      "grad_norm": 2.375173958989858,
+      "learning_rate": 3.72207857193791e-06,
+      "logits/chosen": -0.6621152758598328,
+      "logits/rejected": -0.6223613023757935,
+      "logps/chosen": -0.5641170144081116,
+      "logps/rejected": -0.6517760157585144,
+      "loss": 0.7414,
+      "odds_ratio_loss": 7.36290979385376,
+      "rewards/accuracies": 0.3671875,
+      "rewards/chosen": -0.06517761200666428,
+      "rewards/margins": -0.00876590609550476,
+      "rewards/rejected": -0.056411705911159515,
+      "sft_loss": 0.005075749009847641,
+      "step": 74
+    },
+    {
+      "epoch": 0.411381556393555,
+      "grad_norm": 2.178626918834999,
+      "learning_rate": 3.679819579418414e-06,
+      "logits/chosen": -0.6223607659339905,
+      "logits/rejected": -0.6354334354400635,
+      "logps/chosen": -0.6071732640266418,
+      "logps/rejected": -0.664821207523346,
+      "loss": 0.7537,
+      "odds_ratio_loss": 7.4999518394470215,
+      "rewards/accuracies": 0.3515625,
+      "rewards/chosen": -0.06648211926221848,
+      "rewards/margins": -0.005764788947999477,
+      "rewards/rejected": -0.06071732938289642,
+      "sft_loss": 0.003658156841993332,
+      "step": 75
+    },
+    {
+      "epoch": 0.41686664381213573,
+      "grad_norm": 2.029172062754319,
+      "learning_rate": 3.6371223320885492e-06,
+      "logits/chosen": -0.6456342339515686,
+      "logits/rejected": -0.6101202368736267,
+      "logps/chosen": -0.5387505888938904,
+      "logps/rejected": -0.653649091720581,
+      "loss": 0.748,
+      "odds_ratio_loss": 7.446066379547119,
+      "rewards/accuracies": 0.28125,
+      "rewards/chosen": -0.06536491215229034,
+      "rewards/margins": -0.011489855125546455,
+      "rewards/rejected": -0.05387505888938904,
+      "sft_loss": 0.0033676247112452984,
+      "step": 76
+    },
+    {
+      "epoch": 0.4223517312307165,
+      "grad_norm": 2.566739156357152,
+      "learning_rate": 3.5940026902326825e-06,
+      "logits/chosen": -0.6308640837669373,
+      "logits/rejected": -0.5544230937957764,
+      "logps/chosen": -0.6081095933914185,
+      "logps/rejected": -0.6010054349899292,
+      "loss": 0.686,
+      "odds_ratio_loss": 6.817296028137207,
+      "rewards/accuracies": 0.421875,
+      "rewards/chosen": -0.06010054424405098,
+      "rewards/margins": 0.000710406806319952,
+      "rewards/rejected": -0.06081094965338707,
+      "sft_loss": 0.00426267646253109,
+      "step": 77
+    },
+    {
+      "epoch": 0.42783681864929723,
+      "grad_norm": 2.2168093785628424,
+      "learning_rate": 3.550476671037505e-06,
+      "logits/chosen": -0.6168922781944275,
+      "logits/rejected": -0.6250343322753906,
+      "logps/chosen": -0.7373520135879517,
+      "logps/rejected": -0.6632636785507202,
+      "loss": 0.7541,
+      "odds_ratio_loss": 7.504645824432373,
+      "rewards/accuracies": 0.3671875,
+      "rewards/chosen": -0.06632636487483978,
+      "rewards/margins": 0.0074088433757424355,
+      "rewards/rejected": -0.07373520731925964,
+      "sft_loss": 0.003666748758405447,
+      "step": 78
+    },
+    {
+      "epoch": 0.43332190606787796,
+      "grad_norm": 2.0671981983555767,
+      "learning_rate": 3.5065604426422995e-06,
+      "logits/chosen": -0.6872790455818176,
+      "logits/rejected": -0.6633861064910889,
+      "logps/chosen": -0.6065261363983154,
+      "logps/rejected": -0.6005845069885254,
+      "loss": 0.6867,
+      "odds_ratio_loss": 6.836608409881592,
+      "rewards/accuracies": 0.3984375,
+      "rewards/chosen": -0.06005845218896866,
+      "rewards/margins": 0.0005941606359556317,
+      "rewards/rejected": -0.06065261363983154,
+      "sft_loss": 0.003060600720345974,
+      "step": 79
+    },
+    {
+      "epoch": 0.4388069934864587,
+      "grad_norm": 2.2579388709063477,
+      "learning_rate": 3.462270318133136e-06,
+      "logits/chosen": -0.7127959728240967,
+      "logits/rejected": -0.7076155543327332,
+      "logps/chosen": -0.6201507449150085,
+      "logps/rejected": -0.6548338532447815,
+      "loss": 0.7463,
+      "odds_ratio_loss": 7.421444892883301,
+      "rewards/accuracies": 0.296875,
+      "rewards/chosen": -0.06548339128494263,
+      "rewards/margins": -0.003468305105343461,
+      "rewards/rejected": -0.06201507896184921,
+      "sft_loss": 0.0041392529383301735,
+      "step": 80
+    },
+    {
+      "epoch": 0.4442920809050394,
+      "grad_norm": 2.2666354495836587,
+      "learning_rate": 3.4176227494832305e-06,
+      "logits/chosen": -0.5993680953979492,
+      "logits/rejected": -0.5843874216079712,
+      "logps/chosen": -0.5722631216049194,
+      "logps/rejected": -0.6499100923538208,
+      "loss": 0.7394,
+      "odds_ratio_loss": 7.35059118270874,
+      "rewards/accuracies": 0.34375,
+      "rewards/chosen": -0.06499100476503372,
+      "rewards/margins": -0.007764694280922413,
+      "rewards/rejected": -0.05722631514072418,
+      "sft_loss": 0.004326997324824333,
+      "step": 81
+    },
+    {
+      "epoch": 0.44977716832362014,
+      "grad_norm": 2.283880288724398,
+      "learning_rate": 3.3726343214417023e-06,
+      "logits/chosen": -0.6333982348442078,
+      "logits/rejected": -0.6226303577423096,
+      "logps/chosen": -0.7987644672393799,
+      "logps/rejected": -0.652344822883606,
+      "loss": 0.734,
+      "odds_ratio_loss": 7.291712284088135,
+      "rewards/accuracies": 0.3984375,
+      "rewards/chosen": -0.0652344822883606,
+      "rewards/margins": 0.014641974121332169,
+      "rewards/rejected": -0.07987645268440247,
+      "sft_loss": 0.0048220655880868435,
+      "step": 82
+    },
+    {
+      "epoch": 0.4552622557422009,
+      "grad_norm": 2.1155189965315406,
+      "learning_rate": 3.327321745373021e-06,
+      "logits/chosen": -0.6671404242515564,
+      "logits/rejected": -0.6946146488189697,
+      "logps/chosen": -0.5693039894104004,
+      "logps/rejected": -0.6252968907356262,
+      "loss": 0.7121,
+      "odds_ratio_loss": 7.0859150886535645,
+      "rewards/accuracies": 0.375,
+      "rewards/chosen": -0.06252970546483994,
+      "rewards/margins": -0.005599300377070904,
+      "rewards/rejected": -0.05693039298057556,
+      "sft_loss": 0.003511702874675393,
+      "step": 83
+    },
+    {
+      "epoch": 0.46074734316078164,
+      "grad_norm": 1.9540588513738342,
+      "learning_rate": 3.2817018530494164e-06,
+      "logits/chosen": -0.6938011646270752,
+      "logits/rejected": -0.6819745302200317,
+      "logps/chosen": -0.6028576493263245,
+      "logps/rejected": -0.6305505037307739,
+      "loss": 0.7166,
+      "odds_ratio_loss": 7.133669853210449,
+      "rewards/accuracies": 0.40625,
+      "rewards/chosen": -0.06305506080389023,
+      "rewards/margins": -0.0027692890726029873,
+      "rewards/rejected": -0.060285769402980804,
+      "sft_loss": 0.003260136814787984,
+      "step": 84
+    },
+    {
+      "epoch": 0.46623243057936237,
+      "grad_norm": 2.3068483623141813,
+      "learning_rate": 3.2357915903985605e-06,
+      "logits/chosen": -0.669509768486023,
+      "logits/rejected": -0.6838399171829224,
+      "logps/chosen": -0.7373601198196411,
+      "logps/rejected": -0.63920658826828,
+      "loss": 0.7284,
+      "odds_ratio_loss": 7.239907264709473,
+      "rewards/accuracies": 0.3359375,
+      "rewards/chosen": -0.06392066925764084,
+      "rewards/margins": 0.009815343655645847,
+      "rewards/rejected": -0.07373600453138351,
+      "sft_loss": 0.004396573640406132,
+      "step": 85
+    },
+    {
+      "epoch": 0.4717175179979431,
+      "grad_norm": 2.400639181271588,
+      "learning_rate": 3.1896080112088477e-06,
+      "logits/chosen": -0.6363841891288757,
+      "logits/rejected": -0.6474689245223999,
+      "logps/chosen": -0.6146495938301086,
+      "logps/rejected": -0.665170431137085,
+      "loss": 0.7518,
+      "odds_ratio_loss": 7.468950271606445,
+      "rewards/accuracies": 0.40625,
+      "rewards/chosen": -0.06651704013347626,
+      "rewards/margins": -0.005052081309258938,
+      "rewards/rejected": -0.061464957892894745,
+      "sft_loss": 0.00492935162037611,
+      "step": 86
+    },
+    {
+      "epoch": 0.4772026054165238,
+      "grad_norm": 6.788892165244669,
+      "learning_rate": 3.143168270794612e-06,
+      "logits/chosen": -0.7378703355789185,
+      "logits/rejected": -0.7363454699516296,
+      "logps/chosen": -0.792299747467041,
+      "logps/rejected": -0.6177616119384766,
+      "loss": 0.6952,
+      "odds_ratio_loss": 6.825002670288086,
+      "rewards/accuracies": 0.4453125,
+      "rewards/chosen": -0.061776161193847656,
+      "rewards/margins": 0.017453810200095177,
+      "rewards/rejected": -0.07922996580600739,
+      "sft_loss": 0.012708396650850773,
+      "step": 87
+    },
+    {
+      "epoch": 0.48268769283510454,
+      "grad_norm": 4.194375970507543,
+      "learning_rate": 3.0964896196236217e-06,
+      "logits/chosen": -0.6821984052658081,
+      "logits/rejected": -0.6771599054336548,
+      "logps/chosen": -0.6080144643783569,
+      "logps/rejected": -0.6563798189163208,
+      "loss": 0.7478,
+      "odds_ratio_loss": 7.3482136726379395,
+      "rewards/accuracies": 0.359375,
+      "rewards/chosen": -0.0656379908323288,
+      "rewards/margins": -0.004836536943912506,
+      "rewards/rejected": -0.06080144643783569,
+      "sft_loss": 0.012991908006370068,
+      "step": 88
+    },
+    {
+      "epoch": 0.48817278025368527,
+      "grad_norm": 3.2712874288459868,
+      "learning_rate": 3.0495893969092395e-06,
+      "logits/chosen": -0.7360261082649231,
+      "logits/rejected": -0.648233950138092,
+      "logps/chosen": -0.6474156975746155,
+      "logps/rejected": -0.6273612976074219,
+      "loss": 0.7148,
+      "odds_ratio_loss": 7.092808723449707,
+      "rewards/accuracies": 0.4296875,
+      "rewards/chosen": -0.0627361312508583,
+      "rewards/margins": 0.002005442278459668,
+      "rewards/rejected": -0.0647415742278099,
+      "sft_loss": 0.005477006547152996,
+      "step": 89
+    },
+    {
+      "epoch": 0.49365786767226605,
+      "grad_norm": 3.363992426087972,
+      "learning_rate": 3.0024850241696128e-06,
+      "logits/chosen": -0.6951555013656616,
+      "logits/rejected": -0.6976528167724609,
+      "logps/chosen": -0.6175632476806641,
+      "logps/rejected": -0.6432512402534485,
+      "loss": 0.7269,
+      "odds_ratio_loss": 7.175661087036133,
+      "rewards/accuracies": 0.390625,
+      "rewards/chosen": -0.06432512402534485,
+      "rewards/margins": -0.0025687951128929853,
+      "rewards/rejected": -0.061756327748298645,
+      "sft_loss": 0.009336180053651333,
+      "step": 90
+    },
+    {
+      "epoch": 0.4991429550908468,
+      "grad_norm": 2.4802416219258125,
+      "learning_rate": 2.9551939987562866e-06,
+      "logits/chosen": -0.6312170028686523,
+      "logits/rejected": -0.6572614908218384,
+      "logps/chosen": -0.590150773525238,
+      "logps/rejected": -0.6796594858169556,
+      "loss": 0.7688,
+      "odds_ratio_loss": 7.620222568511963,
+      "rewards/accuracies": 0.3671875,
+      "rewards/chosen": -0.06796594709157944,
+      "rewards/margins": -0.008950873278081417,
+      "rewards/rejected": -0.059015076607465744,
+      "sft_loss": 0.006825140677392483,
+      "step": 91
+    },
+    {
+      "epoch": 0.5046280425094275,
+      "grad_norm": 2.0888605610235156,
+      "learning_rate": 2.907733887354657e-06,
+      "logits/chosen": -0.7382671236991882,
+      "logits/rejected": -0.704073965549469,
+      "logps/chosen": -0.60980224609375,
+      "logps/rejected": -0.5962368249893188,
+      "loss": 0.6756,
+      "odds_ratio_loss": 6.72347354888916,
+      "rewards/accuracies": 0.421875,
+      "rewards/chosen": -0.05962368845939636,
+      "rewards/margins": 0.0013565376866608858,
+      "rewards/rejected": -0.06098022684454918,
+      "sft_loss": 0.0032284611370414495,
+      "step": 92
+    },
+    {
+      "epoch": 0.5101131299280083,
+      "grad_norm": 2.594959628383668,
+      "learning_rate": 2.8601223194586613e-06,
+      "logits/chosen": -0.6301463842391968,
+      "logits/rejected": -0.6508597731590271,
+      "logps/chosen": -0.6760177612304688,
+      "logps/rejected": -0.6208183169364929,
+      "loss": 0.6984,
+      "odds_ratio_loss": 6.931189060211182,
+      "rewards/accuracies": 0.4453125,
+      "rewards/chosen": -0.06208183616399765,
+      "rewards/margins": 0.005519941449165344,
+      "rewards/rejected": -0.067601777613163,
+      "sft_loss": 0.0052444045431911945,
+      "step": 93
+    },
+    {
+      "epoch": 0.515598217346589,
+      "grad_norm": 2.4177461424114957,
+      "learning_rate": 2.8123769808221407e-06,
+      "logits/chosen": -0.7098735570907593,
+      "logits/rejected": -0.694360613822937,
+      "logps/chosen": -0.6068992614746094,
+      "logps/rejected": -0.672624945640564,
+      "loss": 0.7656,
+      "odds_ratio_loss": 7.604451656341553,
+      "rewards/accuracies": 0.3203125,
+      "rewards/chosen": -0.06726250052452087,
+      "rewards/margins": -0.006572564598172903,
+      "rewards/rejected": -0.060689933598041534,
+      "sft_loss": 0.005188749171793461,
+      "step": 94
+    },
+    {
+      "epoch": 0.5210833047651697,
+      "grad_norm": 2.1914520317020125,
+      "learning_rate": 2.7645156068893075e-06,
+      "logits/chosen": -0.6409396529197693,
+      "logits/rejected": -0.6979643702507019,
+      "logps/chosen": -0.534928560256958,
+      "logps/rejected": -0.6406592726707458,
+      "loss": 0.7345,
+      "odds_ratio_loss": 7.306841850280762,
+      "rewards/accuracies": 0.3515625,
+      "rewards/chosen": -0.06406592577695847,
+      "rewards/margins": -0.010573070496320724,
+      "rewards/rejected": -0.05349285155534744,
+      "sft_loss": 0.0038562421686947346,
+      "step": 95
+    },
+    {
+      "epoch": 0.5265683921837504,
+      "grad_norm": 2.0840677499320335,
+      "learning_rate": 2.716555976206748e-06,
+      "logits/chosen": -0.7479691505432129,
+      "logits/rejected": -0.723030149936676,
+      "logps/chosen": -0.5895535349845886,
+      "logps/rejected": -0.6176682114601135,
+      "loss": 0.7002,
+      "odds_ratio_loss": 6.960841655731201,
+      "rewards/accuracies": 0.484375,
+      "rewards/chosen": -0.06176682561635971,
+      "rewards/margins": -0.0028114626184105873,
+      "rewards/rejected": -0.0589553564786911,
+      "sft_loss": 0.004080518614500761,
+      "step": 96
+    },
+    {
+      "epoch": 0.5320534796023312,
+      "grad_norm": 2.444265846509054,
+      "learning_rate": 2.6685159038194202e-06,
+      "logits/chosen": -0.6860970258712769,
+      "logits/rejected": -0.6489076018333435,
+      "logps/chosen": -0.61269611120224,
+      "logps/rejected": -0.6757325530052185,
+      "loss": 0.7617,
+      "odds_ratio_loss": 7.5671162605285645,
+      "rewards/accuracies": 0.3359375,
+      "rewards/chosen": -0.06757325679063797,
+      "rewards/margins": -0.006303644739091396,
+      "rewards/rejected": -0.061269611120224,
+      "sft_loss": 0.0049641747027635574,
+      "step": 97
+    },
+    {
+      "epoch": 0.5375385670209119,
+      "grad_norm": 2.128773042167142,
+      "learning_rate": 2.6204132346530936e-06,
+      "logits/chosen": -0.7502190470695496,
+      "logits/rejected": -0.7040455937385559,
+      "logps/chosen": -0.7278321981430054,
+      "logps/rejected": -0.6399706602096558,
+      "loss": 0.7272,
+      "odds_ratio_loss": 7.2354841232299805,
+      "rewards/accuracies": 0.390625,
+      "rewards/chosen": -0.0639970600605011,
+      "rewards/margins": 0.008786162361502647,
+      "rewards/rejected": -0.0727832242846489,
+      "sft_loss": 0.00368284760043025,
+      "step": 98
+    },
+    {
+      "epoch": 0.5430236544394926,
+      "grad_norm": 2.5919828332695833,
+      "learning_rate": 2.572265836885682e-06,
+      "logits/chosen": -0.7643656134605408,
+      "logits/rejected": -0.6870676279067993,
+      "logps/chosen": -0.7556079030036926,
+      "logps/rejected": -0.6754346489906311,
+      "loss": 0.7635,
+      "odds_ratio_loss": 7.5766706466674805,
+      "rewards/accuracies": 0.3671875,
+      "rewards/chosen": -0.06754346191883087,
+      "rewards/margins": 0.008017327636480331,
+      "rewards/rejected": -0.0755607932806015,
+      "sft_loss": 0.005802985280752182,
+      "step": 99
+    },
+    {
+      "epoch": 0.5485087418580734,
+      "grad_norm": 2.449701609589594,
+      "learning_rate": 2.524091595309952e-06,
+      "logits/chosen": -0.701339602470398,
+      "logits/rejected": -0.6497517824172974,
+      "logps/chosen": -0.6323676705360413,
+      "logps/rejected": -0.633608341217041,
+      "loss": 0.7136,
+      "odds_ratio_loss": 7.087641716003418,
+      "rewards/accuracies": 0.4375,
+      "rewards/chosen": -0.06336082518100739,
+      "rewards/margins": -0.00012405950110405684,
+      "rewards/rejected": -0.0632367730140686,
+      "sft_loss": 0.004797851666808128,
+      "step": 100
+    },
+    {
+      "epoch": 0.5539938292766541,
+      "grad_norm": 1.8980955014875789,
+      "learning_rate": 2.475908404690049e-06,
+      "logits/chosen": -0.7044223546981812,
+      "logits/rejected": -0.6829949021339417,
+      "logps/chosen": -0.5901881456375122,
+      "logps/rejected": -0.5841273665428162,
+      "loss": 0.6646,
+      "odds_ratio_loss": 6.614431381225586,
+      "rewards/accuracies": 0.453125,
+      "rewards/chosen": -0.05841274932026863,
+      "rewards/margins": 0.0006060737650841475,
+      "rewards/rejected": -0.0590188167989254,
+      "sft_loss": 0.0031673426274210215,
+      "step": 101
+    },
+    {
+      "epoch": 0.5594789166952349,
+      "grad_norm": 2.283805508955311,
+      "learning_rate": 2.427734163114319e-06,
+      "logits/chosen": -0.7973791360855103,
+      "logits/rejected": -0.7453299164772034,
+      "logps/chosen": -0.5993779301643372,
+      "logps/rejected": -0.6565759778022766,
+      "loss": 0.7469,
+      "odds_ratio_loss": 7.423154354095459,
+      "rewards/accuracies": 0.34375,
+      "rewards/chosen": -0.0656576007604599,
+      "rewards/margins": -0.0057198042050004005,
+      "rewards/rejected": -0.059937797486782074,
+      "sft_loss": 0.004618693143129349,
+      "step": 102
+    },
+    {
+      "epoch": 0.5649640041138155,
+      "grad_norm": 2.1097973408223165,
+      "learning_rate": 2.3795867653469072e-06,
+      "logits/chosen": -0.7150146961212158,
+      "logits/rejected": -0.6962917447090149,
+      "logps/chosen": -0.6036943793296814,
+      "logps/rejected": -0.6037816405296326,
+      "loss": 0.6859,
+      "odds_ratio_loss": 6.8253865242004395,
+      "rewards/accuracies": 0.390625,
+      "rewards/chosen": -0.06037816405296326,
+      "rewards/margins": -8.72323289513588e-06,
+      "rewards/rejected": -0.06036944314837456,
+      "sft_loss": 0.0033163288608193398,
+      "step": 103
+    },
+    {
+      "epoch": 0.5704490915323963,
+      "grad_norm": 2.261348542854153,
+      "learning_rate": 2.3314840961805806e-06,
+      "logits/chosen": -0.7404282689094543,
+      "logits/rejected": -0.7224586606025696,
+      "logps/chosen": -0.6297097206115723,
+      "logps/rejected": -0.6101536154747009,
+      "loss": 0.6902,
+      "odds_ratio_loss": 6.866119384765625,
+      "rewards/accuracies": 0.40625,
+      "rewards/chosen": -0.06101536750793457,
+      "rewards/margins": 0.0019556120969355106,
+      "rewards/rejected": -0.06297097355127335,
+      "sft_loss": 0.0035803657956421375,
+      "step": 104
+    },
+    {
+      "epoch": 0.575934178950977,
+      "grad_norm": 2.3462527146244394,
+      "learning_rate": 2.2834440237932537e-06,
+      "logits/chosen": -0.7421097755432129,
+      "logits/rejected": -0.7392350435256958,
+      "logps/chosen": -0.5688321590423584,
+      "logps/rejected": -0.5941362380981445,
+      "loss": 0.676,
+      "odds_ratio_loss": 6.721547603607178,
+      "rewards/accuracies": 0.4375,
+      "rewards/chosen": -0.05941362306475639,
+      "rewards/margins": -0.0025304073933511972,
+      "rewards/rejected": -0.05688321590423584,
+      "sft_loss": 0.003852433292195201,
+      "step": 105
+    },
+    {
+      "epoch": 0.5814192663695578,
+      "grad_norm": 2.2368897755771844,
+      "learning_rate": 2.2354843931106933e-06,
+      "logits/chosen": -0.6827410459518433,
+      "logits/rejected": -0.6356707811355591,
+      "logps/chosen": -0.7240877747535706,
+      "logps/rejected": -0.6512930393218994,
+      "loss": 0.7339,
+      "odds_ratio_loss": 7.29488468170166,
+      "rewards/accuracies": 0.3984375,
+      "rewards/chosen": -0.06512930989265442,
+      "rewards/margins": 0.007279472425580025,
+      "rewards/rejected": -0.0724087804555893,
+      "sft_loss": 0.004439079202711582,
+      "step": 106
+    },
+    {
+      "epoch": 0.5869043537881385,
+      "grad_norm": 3.1022172612718313,
+      "learning_rate": 2.1876230191778598e-06,
+      "logits/chosen": -0.749907374382019,
+      "logits/rejected": -0.7236763834953308,
+      "logps/chosen": -0.6079933047294617,
+      "logps/rejected": -0.6530497074127197,
+      "loss": 0.7391,
+      "odds_ratio_loss": 7.280797481536865,
+      "rewards/accuracies": 0.3828125,
+      "rewards/chosen": -0.06530497968196869,
+      "rewards/margins": -0.004505641758441925,
+      "rewards/rejected": -0.06079933047294617,
+      "sft_loss": 0.01101373415440321,
+      "step": 107
+    },
+    {
+      "epoch": 0.5923894412067192,
+      "grad_norm": 2.197063400051544,
+      "learning_rate": 2.13987768054134e-06,
+      "logits/chosen": -0.6889519095420837,
+      "logits/rejected": -0.6868714094161987,
+      "logps/chosen": -0.6039581894874573,
+      "logps/rejected": -0.5748084187507629,
+      "loss": 0.6528,
+      "odds_ratio_loss": 6.491232395172119,
+      "rewards/accuracies": 0.4296875,
+      "rewards/chosen": -0.057480841875076294,
+      "rewards/margins": 0.00291498308070004,
+      "rewards/rejected": -0.060395821928977966,
+      "sft_loss": 0.00366477994248271,
+      "step": 108
+    },
+    {
+      "epoch": 0.5978745286253,
+      "grad_norm": 2.159407846285504,
+      "learning_rate": 2.0922661126453436e-06,
+      "logits/chosen": -0.6941989064216614,
+      "logits/rejected": -0.6537268757820129,
+      "logps/chosen": -0.7240587472915649,
+      "logps/rejected": -0.6288081407546997,
+      "loss": 0.7097,
+      "odds_ratio_loss": 7.062224864959717,
+      "rewards/accuracies": 0.453125,
+      "rewards/chosen": -0.06288080662488937,
+      "rewards/margins": 0.00952506810426712,
+      "rewards/rejected": -0.0724058747291565,
+      "sft_loss": 0.0034658340737223625,
+      "step": 109
+    },
+    {
+      "epoch": 0.6033596160438807,
+      "grad_norm": 2.4398874181778822,
+      "learning_rate": 2.044806001243714e-06,
+      "logits/chosen": -0.7023935317993164,
+      "logits/rejected": -0.6535657644271851,
+      "logps/chosen": -0.6954001188278198,
+      "logps/rejected": -0.6107276082038879,
+      "loss": 0.6873,
+      "odds_ratio_loss": 6.822115898132324,
+      "rewards/accuracies": 0.4921875,
+      "rewards/chosen": -0.061072759330272675,
+      "rewards/margins": 0.008467256091535091,
+      "rewards/rejected": -0.06954001635313034,
+      "sft_loss": 0.00511753186583519,
+      "step": 110
+    },
+    {
+      "epoch": 0.6088447034624614,
+      "grad_norm": 3.1443922922473795,
+      "learning_rate": 1.9975149758303885e-06,
+      "logits/chosen": -0.6652725338935852,
+      "logits/rejected": -0.6267548203468323,
+      "logps/chosen": -0.7502924799919128,
+      "logps/rejected": -0.6726891398429871,
+      "loss": 0.7606,
+      "odds_ratio_loss": 7.564757347106934,
+      "rewards/accuracies": 0.34375,
+      "rewards/chosen": -0.06726891547441483,
+      "rewards/margins": 0.007760328706353903,
+      "rewards/rejected": -0.07502924650907516,
+      "sft_loss": 0.004097915254533291,
+      "step": 111
+    },
+    {
+      "epoch": 0.6143297908810421,
+      "grad_norm": 3.640891877033526,
+      "learning_rate": 1.9504106030907605e-06,
+      "logits/chosen": -0.6462224125862122,
+      "logits/rejected": -0.661245584487915,
+      "logps/chosen": -0.6592689752578735,
+      "logps/rejected": -0.6112386584281921,
+      "loss": 0.6912,
+      "odds_ratio_loss": 6.873685836791992,
+      "rewards/accuracies": 0.453125,
+      "rewards/chosen": -0.06112387031316757,
+      "rewards/margins": 0.00480302982032299,
+      "rewards/rejected": -0.06592689454555511,
+      "sft_loss": 0.0038500106893479824,
+      "step": 112
+    },
+    {
+      "epoch": 0.6198148782996229,
+      "grad_norm": 2.7204380499722967,
+      "learning_rate": 1.9035103803763793e-06,
+      "logits/chosen": -0.6889287233352661,
+      "logits/rejected": -0.7191394567489624,
+      "logps/chosen": -0.6261990070343018,
+      "logps/rejected": -0.6758837699890137,
+      "loss": 0.7625,
+      "odds_ratio_loss": 7.566086769104004,
+      "rewards/accuracies": 0.359375,
+      "rewards/chosen": -0.06758838146924973,
+      "rewards/margins": -0.004968480207026005,
+      "rewards/rejected": -0.0626199021935463,
+      "sft_loss": 0.005911496467888355,
+      "step": 113
+    },
+    {
+      "epoch": 0.6252999657182037,
+      "grad_norm": 2.3221228206896543,
+      "learning_rate": 1.8568317292053894e-06,
+      "logits/chosen": -0.6986726522445679,
+      "logits/rejected": -0.7248036861419678,
+      "logps/chosen": -0.7008225917816162,
+      "logps/rejected": -0.6063806414604187,
+      "loss": 0.6896,
+      "odds_ratio_loss": 6.8465399742126465,
+      "rewards/accuracies": 0.4296875,
+      "rewards/chosen": -0.06063806638121605,
+      "rewards/margins": 0.009444191120564938,
+      "rewards/rejected": -0.07008224725723267,
+      "sft_loss": 0.004963008686900139,
+      "step": 114
+    },
+    {
+      "epoch": 0.6307850531367843,
+      "grad_norm": 2.3323657507043585,
+      "learning_rate": 1.8103919887911525e-06,
+      "logits/chosen": -0.6923948526382446,
+      "logits/rejected": -0.6713389158248901,
+      "logps/chosen": -0.6527585983276367,
+      "logps/rejected": -0.6079282164573669,
+      "loss": 0.6887,
+      "odds_ratio_loss": 6.84266996383667,
+      "rewards/accuracies": 0.421875,
+      "rewards/chosen": -0.06079282611608505,
+      "rewards/margins": 0.004483034834265709,
+      "rewards/rejected": -0.06527585536241531,
+      "sft_loss": 0.004444844089448452,
+      "step": 115
+    },
+    {
+      "epoch": 0.6362701405553651,
+      "grad_norm": 2.2756656865836824,
+      "learning_rate": 1.7642084096014405e-06,
+      "logits/chosen": -0.6769150495529175,
+      "logits/rejected": -0.6519336104393005,
+      "logps/chosen": -0.6810208559036255,
+      "logps/rejected": -0.6118040680885315,
+      "loss": 0.6967,
+      "odds_ratio_loss": 6.925226211547852,
+      "rewards/accuracies": 0.359375,
+      "rewards/chosen": -0.06118040531873703,
+      "rewards/margins": 0.006921680178493261,
+      "rewards/rejected": -0.06810208410024643,
+      "sft_loss": 0.0041780658066272736,
+      "step": 116
+    },
+    {
+      "epoch": 0.6417552279739458,
+      "grad_norm": 4.533574316801059,
+      "learning_rate": 1.718298146950585e-06,
+      "logits/chosen": -0.6806620955467224,
+      "logits/rejected": -0.6420994400978088,
+      "logps/chosen": -0.6579238176345825,
+      "logps/rejected": -0.6758315563201904,
+      "loss": 0.7643,
+      "odds_ratio_loss": 7.476547718048096,
+      "rewards/accuracies": 0.390625,
+      "rewards/chosen": -0.06758316606283188,
+      "rewards/margins": -0.0017907717265188694,
+      "rewards/rejected": -0.06579238921403885,
+      "sft_loss": 0.016644245013594627,
+      "step": 117
+    },
+    {
+      "epoch": 0.6472403153925266,
+      "grad_norm": 2.4121811138925016,
+      "learning_rate": 1.6726782546269793e-06,
+      "logits/chosen": -0.7141998410224915,
+      "logits/rejected": -0.6570146083831787,
+      "logps/chosen": -0.6056728959083557,
+      "logps/rejected": -0.6163904666900635,
+      "loss": 0.7006,
+      "odds_ratio_loss": 6.9596381187438965,
+      "rewards/accuracies": 0.375,
+      "rewards/chosen": -0.061639051884412766,
+      "rewards/margins": -0.0010717544937506318,
+      "rewards/rejected": -0.06056729331612587,
+      "sft_loss": 0.004627158399671316,
+      "step": 118
+    },
+    {
+      "epoch": 0.6527254028111072,
+      "grad_norm": 2.116647341589779,
+      "learning_rate": 1.6273656785582986e-06,
+      "logits/chosen": -0.7080201506614685,
+      "logits/rejected": -0.6555320024490356,
+      "logps/chosen": -0.6334236860275269,
+      "logps/rejected": -0.6177656650543213,
+      "loss": 0.7001,
+      "odds_ratio_loss": 6.9610371589660645,
+      "rewards/accuracies": 0.421875,
+      "rewards/chosen": -0.06177656352519989,
+      "rewards/margins": 0.0015658115735277534,
+      "rewards/rejected": -0.0633423700928688,
+      "sft_loss": 0.00395183265209198,
+      "step": 119
+    },
+    {
+      "epoch": 0.658210490229688,
+      "grad_norm": 2.10163675871464,
+      "learning_rate": 1.58237725051677e-06,
+      "logits/chosen": -0.6549379229545593,
+      "logits/rejected": -0.6431285738945007,
+      "logps/chosen": -0.5666630268096924,
+      "logps/rejected": -0.534731388092041,
+      "loss": 0.6123,
+      "odds_ratio_loss": 6.093177795410156,
+      "rewards/accuracies": 0.453125,
+      "rewards/chosen": -0.05347314849495888,
+      "rewards/margins": 0.0031931637786328793,
+      "rewards/rejected": -0.056666307151317596,
+      "sft_loss": 0.003024790436029434,
+      "step": 120
+    },
+    {
+      "epoch": 0.6636955776482688,
+      "grad_norm": 2.117301914309323,
+      "learning_rate": 1.5377296818668638e-06,
+      "logits/chosen": -0.6497895121574402,
+      "logits/rejected": -0.6862410306930542,
+      "logps/chosen": -0.5236161351203918,
+      "logps/rejected": -0.6108285188674927,
+      "loss": 0.7019,
+      "odds_ratio_loss": 6.986342430114746,
+      "rewards/accuracies": 0.3515625,
+      "rewards/chosen": -0.06108284741640091,
+      "rewards/margins": -0.008721234276890755,
+      "rewards/rejected": -0.052361615002155304,
+      "sft_loss": 0.0032780070323497057,
+      "step": 121
+    },
+    {
+      "epoch": 0.6691806650668495,
+      "grad_norm": 2.2927939430967093,
+      "learning_rate": 1.4934395573577016e-06,
+      "logits/chosen": -0.680001974105835,
+      "logits/rejected": -0.6553654670715332,
+      "logps/chosen": -0.5565463900566101,
+      "logps/rejected": -0.5977818965911865,
+      "loss": 0.681,
+      "odds_ratio_loss": 6.767004489898682,
+      "rewards/accuracies": 0.4140625,
+      "rewards/chosen": -0.05977818742394447,
+      "rewards/margins": -0.004123550374060869,
+      "rewards/rejected": -0.05565463379025459,
+      "sft_loss": 0.004254105966538191,
+      "step": 122
+    },
+    {
+      "epoch": 0.6746657524854303,
+      "grad_norm": 1.9919891401234977,
+      "learning_rate": 1.449523328962496e-06,
+      "logits/chosen": -0.6618916988372803,
+      "logits/rejected": -0.6230502724647522,
+      "logps/chosen": -0.6067531108856201,
+      "logps/rejected": -0.6063209772109985,
+      "loss": 0.6881,
+      "odds_ratio_loss": 6.851033687591553,
+      "rewards/accuracies": 0.40625,
+      "rewards/chosen": -0.06063209846615791,
+      "rewards/margins": 4.3215928599238396e-05,
+      "rewards/rejected": -0.06067531555891037,
+      "sft_loss": 0.0030309129506349564,
+      "step": 123
+    },
+    {
+      "epoch": 0.6801508399040109,
+      "grad_norm": 2.4998193508161006,
+      "learning_rate": 1.4059973097673187e-06,
+      "logits/chosen": -0.7269992232322693,
+      "logits/rejected": -0.6942679286003113,
+      "logps/chosen": -0.6811712980270386,
+      "logps/rejected": -0.6196709871292114,
+      "loss": 0.6976,
+      "odds_ratio_loss": 6.929691314697266,
+      "rewards/accuracies": 0.453125,
+      "rewards/chosen": -0.061967093497514725,
+      "rewards/margins": 0.006150041241198778,
+      "rewards/rejected": -0.06811713427305222,
+      "sft_loss": 0.004657902754843235,
+      "step": 124
+    },
+    {
+      "epoch": 0.6856359273225917,
+      "grad_norm": 2.1081004296697583,
+      "learning_rate": 1.3628776679114516e-06,
+      "logits/chosen": -0.7377051711082458,
+      "logits/rejected": -0.7069607973098755,
+      "logps/chosen": -0.8355605602264404,
+      "logps/rejected": -0.6344387531280518,
+      "loss": 0.7112,
+      "odds_ratio_loss": 7.074479103088379,
+      "rewards/accuracies": 0.4765625,
+      "rewards/chosen": -0.0634438693523407,
+      "rewards/margins": 0.020112188532948494,
+      "rewards/rejected": -0.08355606347322464,
+      "sft_loss": 0.003787390887737274,
+      "step": 125
+    },
+    {
+      "epoch": 0.6911210147411725,
+      "grad_norm": 2.085036928586896,
+      "learning_rate": 1.3201804205815872e-06,
+      "logits/chosen": -0.6863076686859131,
+      "logits/rejected": -0.6810216307640076,
+      "logps/chosen": -0.7168741226196289,
+      "logps/rejected": -0.6459383964538574,
+      "loss": 0.7361,
+      "odds_ratio_loss": 7.325974464416504,
+      "rewards/accuracies": 0.3515625,
+      "rewards/chosen": -0.0645938366651535,
+      "rewards/margins": 0.0070935748517513275,
+      "rewards/rejected": -0.07168740779161453,
+      "sft_loss": 0.0035439669154584408,
+      "step": 126
+    },
+    {
+      "epoch": 0.6966061021597532,
+      "grad_norm": 2.351506031335298,
+      "learning_rate": 1.277921428062091e-06,
+      "logits/chosen": -0.6885042190551758,
+      "logits/rejected": -0.6516430974006653,
+      "logps/chosen": -0.6768959164619446,
+      "logps/rejected": -0.6008927822113037,
+      "loss": 0.6796,
+      "odds_ratio_loss": 6.754604339599609,
+      "rewards/accuracies": 0.484375,
+      "rewards/chosen": -0.06008927896618843,
+      "rewards/margins": 0.007600321434438229,
+      "rewards/rejected": -0.06768959760665894,
+      "sft_loss": 0.004147130064666271,
+      "step": 127
+    },
+    {
+      "epoch": 0.7020911895783339,
+      "grad_norm": 2.027558705672352,
+      "learning_rate": 1.2361163878435594e-06,
+      "logits/chosen": -0.6875728964805603,
+      "logits/rejected": -0.6000896096229553,
+      "logps/chosen": -0.6457569599151611,
+      "logps/rejected": -0.6216841340065002,
+      "loss": 0.7047,
+      "odds_ratio_loss": 7.014739036560059,
+      "rewards/accuracies": 0.40625,
+      "rewards/chosen": -0.062168411910533905,
+      "rewards/margins": 0.0024072853848338127,
+      "rewards/rejected": -0.06457570195198059,
+      "sft_loss": 0.0032098242081701756,
+      "step": 128
+    },
+    {
+      "epoch": 0.7075762769969146,
+      "grad_norm": 2.6891394023374913,
+      "learning_rate": 1.1947808287918406e-06,
+      "logits/chosen": -0.7073705792427063,
+      "logits/rejected": -0.6391410231590271,
+      "logps/chosen": -0.9505314826965332,
+      "logps/rejected": -0.6497754454612732,
+      "loss": 0.7381,
+      "odds_ratio_loss": 7.3174543380737305,
+      "rewards/accuracies": 0.3359375,
+      "rewards/chosen": -0.06497755646705627,
+      "rewards/margins": 0.030075591057538986,
+      "rewards/rejected": -0.09505314379930496,
+      "sft_loss": 0.006380516104400158,
+      "step": 129
+    },
+    {
+      "epoch": 0.7130613644154954,
+      "grad_norm": 2.7243982781242653,
+      "learning_rate": 1.153930105379695e-06,
+      "logits/chosen": -0.6403245329856873,
+      "logits/rejected": -0.6941784620285034,
+      "logps/chosen": -0.6206578016281128,
+      "logps/rejected": -0.6329513788223267,
+      "loss": 0.7166,
+      "odds_ratio_loss": 7.099569797515869,
+      "rewards/accuracies": 0.4140625,
+      "rewards/chosen": -0.0632951408624649,
+      "rewards/margins": -0.0012293587205931544,
+      "rewards/rejected": -0.06206577643752098,
+      "sft_loss": 0.006686141714453697,
+      "step": 130
+    },
+    {
+      "epoch": 0.7185464518340761,
+      "grad_norm": 3.0501747152037226,
+      "learning_rate": 1.1135793919832336e-06,
+      "logits/chosen": -0.6880025863647461,
+      "logits/rejected": -0.6472803950309753,
+      "logps/chosen": -0.6972535252571106,
+      "logps/rejected": -0.6315404176712036,
+      "loss": 0.7149,
+      "odds_ratio_loss": 7.093357086181641,
+      "rewards/accuracies": 0.40625,
+      "rewards/chosen": -0.06315404176712036,
+      "rewards/margins": 0.006571306847035885,
+      "rewards/rejected": -0.06972534954547882,
+      "sft_loss": 0.005599521566182375,
+      "step": 131
+    },
+    {
+      "epoch": 0.7240315392526568,
+      "grad_norm": 4.097059851270684,
+      "learning_rate": 1.0737436772452602e-06,
+      "logits/chosen": -0.6801282167434692,
+      "logits/rejected": -0.6225499510765076,
+      "logps/chosen": -0.6466732025146484,
+      "logps/rejected": -0.6455317139625549,
+      "loss": 0.7287,
+      "odds_ratio_loss": 7.189926624298096,
+      "rewards/accuracies": 0.4296875,
+      "rewards/chosen": -0.0645531713962555,
+      "rewards/margins": 0.00011415383778512478,
+      "rewards/rejected": -0.06466732174158096,
+      "sft_loss": 0.00968949869275093,
+      "step": 132
+    },
+    {
+      "epoch": 0.7295166266712376,
+      "grad_norm": 2.1627454720493695,
+      "learning_rate": 1.0344377585076e-06,
+      "logits/chosen": -0.6447775363922119,
+      "logits/rejected": -0.6088653206825256,
+      "logps/chosen": -0.5911606550216675,
+      "logps/rejected": -0.5996113419532776,
+      "loss": 0.6819,
+      "odds_ratio_loss": 6.785092830657959,
+      "rewards/accuracies": 0.3671875,
+      "rewards/chosen": -0.05996113270521164,
+      "rewards/margins": -0.000845066097099334,
+      "rewards/rejected": -0.05911606550216675,
+      "sft_loss": 0.00336868641898036,
+      "step": 133
+    },
+    {
+      "epoch": 0.7350017140898183,
+      "grad_norm": 2.3398426117994826,
+      "learning_rate": 9.956762363144892e-07,
+      "logits/chosen": -0.6249683499336243,
+      "logits/rejected": -0.6248048543930054,
+      "logps/chosen": -0.6350124478340149,
+      "logps/rejected": -0.5843938589096069,
+      "loss": 0.666,
+      "odds_ratio_loss": 6.617175579071045,
+      "rewards/accuracies": 0.4453125,
+      "rewards/chosen": -0.05843937769532204,
+      "rewards/margins": 0.005061861127614975,
+      "rewards/rejected": -0.06350124627351761,
+      "sft_loss": 0.004313306882977486,
+      "step": 134
+    },
+    {
+      "epoch": 0.7404868015083991,
+      "grad_norm": 2.241323764733524,
+      "learning_rate": 9.574735089890765e-07,
+      "logits/chosen": -0.6376866102218628,
+      "logits/rejected": -0.5942208766937256,
+      "logps/chosen": -0.6546105742454529,
+      "logps/rejected": -0.6053332090377808,
+      "loss": 0.6863,
+      "odds_ratio_loss": 6.821003437042236,
+      "rewards/accuracies": 0.3828125,
+      "rewards/chosen": -0.060533322393894196,
+      "rewards/margins": 0.0049277422949671745,
+      "rewards/rejected": -0.06546106189489365,
+      "sft_loss": 0.004214235581457615,
+      "step": 135
+    },
+    {
+      "epoch": 0.7459718889269797,
+      "grad_norm": 2.2084551304631654,
+      "learning_rate": 9.198437672850249e-07,
+      "logits/chosen": -0.6654888391494751,
+      "logits/rejected": -0.616665244102478,
+      "logps/chosen": -0.6150586009025574,
+      "logps/rejected": -0.6242785453796387,
+      "loss": 0.7088,
+      "odds_ratio_loss": 7.0509033203125,
+      "rewards/accuracies": 0.3984375,
+      "rewards/chosen": -0.062427863478660583,
+      "rewards/margins": -0.0009219995117746294,
+      "rewards/rejected": -0.06150586158037186,
+      "sft_loss": 0.0037482583429664373,
+      "step": 136
+    },
+    {
+      "epoch": 0.7514569763455605,
+      "grad_norm": 2.5791794615739603,
+      "learning_rate": 8.828009891152301e-07,
+      "logits/chosen": -0.6492202877998352,
+      "logits/rejected": -0.6373860836029053,
+      "logps/chosen": -0.6348594427108765,
+      "logps/rejected": -0.5977117419242859,
+      "loss": 0.6816,
+      "odds_ratio_loss": 6.770447731018066,
+      "rewards/accuracies": 0.390625,
+      "rewards/chosen": -0.05977117270231247,
+      "rewards/margins": 0.003714768448844552,
+      "rewards/rejected": -0.06348594278097153,
+      "sft_loss": 0.004584567621350288,
+      "step": 137
+    },
+    {
+      "epoch": 0.7569420637641412,
+      "grad_norm": 2.10960114347503,
+      "learning_rate": 8.463589343595976e-07,
+      "logits/chosen": -0.6831432580947876,
+      "logits/rejected": -0.6670156717300415,
+      "logps/chosen": -0.6033125519752502,
+      "logps/rejected": -0.589252769947052,
+      "loss": 0.6723,
+      "odds_ratio_loss": 6.688982009887695,
+      "rewards/accuracies": 0.40625,
+      "rewards/chosen": -0.05892528221011162,
+      "rewards/margins": 0.0014059704262763262,
+      "rewards/rejected": -0.06033124774694443,
+      "sft_loss": 0.0033750347793102264,
+      "step": 138
+    },
+    {
+      "epoch": 0.762427151182722,
+      "grad_norm": 2.2233884047037513,
+      "learning_rate": 8.105311397538085e-07,
+      "logits/chosen": -0.6301755905151367,
+      "logits/rejected": -0.6074115037918091,
+      "logps/chosen": -0.642350435256958,
+      "logps/rejected": -0.6004490852355957,
+      "loss": 0.6787,
+      "odds_ratio_loss": 6.747027397155762,
+      "rewards/accuracies": 0.453125,
+      "rewards/chosen": -0.06004491448402405,
+      "rewards/margins": 0.004190134350210428,
+      "rewards/rejected": -0.06423504650592804,
+      "sft_loss": 0.0040018935687839985,
+      "step": 139
+    },
+    {
+      "epoch": 0.7679122386013028,
+      "grad_norm": 2.1001393729716997,
+      "learning_rate": 7.753309138609705e-07,
+      "logits/chosen": -0.6419239044189453,
+      "logits/rejected": -0.6177327632904053,
+      "logps/chosen": -0.7455356121063232,
+      "logps/rejected": -0.6029459834098816,
+      "loss": 0.6764,
+      "odds_ratio_loss": 6.730374336242676,
+      "rewards/accuracies": 0.4609375,
+      "rewards/chosen": -0.06029459461569786,
+      "rewards/margins": 0.014258962124586105,
+      "rewards/rejected": -0.07455356419086456,
+      "sft_loss": 0.003380353096872568,
+      "step": 140
+    },
+    {
+      "epoch": 0.7733973260198834,
+      "grad_norm": 1.955388109291546,
+      "learning_rate": 7.407713321280377e-07,
+      "logits/chosen": -0.6610326766967773,
+      "logits/rejected": -0.5916581749916077,
+      "logps/chosen": -0.6852571368217468,
+      "logps/rejected": -0.5728943347930908,
+      "loss": 0.6502,
+      "odds_ratio_loss": 6.471943378448486,
+      "rewards/accuracies": 0.453125,
+      "rewards/chosen": -0.057289429008960724,
+      "rewards/margins": 0.011236282996833324,
+      "rewards/rejected": -0.06852570921182632,
+      "sft_loss": 0.003054150380194187,
+      "step": 141
+    },
+    {
+      "epoch": 0.7788824134384642,
+      "grad_norm": 1.9633538931591568,
+      "learning_rate": 7.068652320288081e-07,
+      "logits/chosen": -0.5979795455932617,
+      "logits/rejected": -0.5797224640846252,
+      "logps/chosen": -0.5880254507064819,
+      "logps/rejected": -0.6247429847717285,
+      "loss": 0.7109,
+      "odds_ratio_loss": 7.07578706741333,
+      "rewards/accuracies": 0.375,
+      "rewards/chosen": -0.06247429549694061,
+      "rewards/margins": -0.003671749262139201,
+      "rewards/rejected": -0.05880254879593849,
+      "sft_loss": 0.003300985088571906,
+      "step": 142
+    },
+    {
+      "epoch": 0.7843675008570449,
+      "grad_norm": 1.9235465695795355,
+      "learning_rate": 6.736252082953307e-07,
+      "logits/chosen": -0.5846747756004333,
+      "logits/rejected": -0.5656622052192688,
+      "logps/chosen": -0.5795245170593262,
+      "logps/rejected": -0.5894888639450073,
+      "loss": 0.6764,
+      "odds_ratio_loss": 6.733616352081299,
+      "rewards/accuracies": 0.359375,
+      "rewards/chosen": -0.05894888564944267,
+      "rewards/margins": -0.0009964264463633299,
+      "rewards/rejected": -0.057952456176280975,
+      "sft_loss": 0.0030053844675421715,
+      "step": 143
+    },
+    {
+      "epoch": 0.7898525882756257,
+      "grad_norm": 1.8461477291438253,
+      "learning_rate": 6.410636082394772e-07,
+      "logits/chosen": -0.6314021944999695,
+      "logits/rejected": -0.6141982078552246,
+      "logps/chosen": -0.7406495809555054,
+      "logps/rejected": -0.6059472560882568,
+      "loss": 0.6888,
+      "odds_ratio_loss": 6.85874605178833,
+      "rewards/accuracies": 0.40625,
+      "rewards/chosen": -0.06059472635388374,
+      "rewards/margins": 0.013470232486724854,
+      "rewards/rejected": -0.0740649551153183,
+      "sft_loss": 0.0029607657343149185,
+      "step": 144
+    },
+    {
+      "epoch": 0.7953376756942063,
+      "grad_norm": 2.4596952552330196,
+      "learning_rate": 6.091925271664156e-07,
+      "logits/chosen": -0.6452285051345825,
+      "logits/rejected": -0.6411218047142029,
+      "logps/chosen": -0.6096831560134888,
+      "logps/rejected": -0.6008501648902893,
+      "loss": 0.6837,
+      "odds_ratio_loss": 6.7790327072143555,
+      "rewards/accuracies": 0.4375,
+      "rewards/chosen": -0.06008501723408699,
+      "rewards/margins": 0.0008833012543618679,
+      "rewards/rejected": -0.0609683133661747,
+      "sft_loss": 0.005798771046102047,
+      "step": 145
+    },
+    {
+      "epoch": 0.8008227631127871,
+      "grad_norm": 2.395026025025573,
+      "learning_rate": 5.780238038817035e-07,
+      "logits/chosen": -0.6628804802894592,
+      "logits/rejected": -0.6211153268814087,
+      "logps/chosen": -0.6752290725708008,
+      "logps/rejected": -0.6579370498657227,
+      "loss": 0.746,
+      "odds_ratio_loss": 7.417111873626709,
+      "rewards/accuracies": 0.328125,
+      "rewards/chosen": -0.0657937079668045,
+      "rewards/margins": 0.0017291962867602706,
+      "rewards/rejected": -0.06752290576696396,
+      "sft_loss": 0.004298758693039417,
+      "step": 146
+    },
+    {
+      "epoch": 0.8063078505313679,
+      "grad_norm": 2.068579141907181,
+      "learning_rate": 5.47569016293649e-07,
+      "logits/chosen": -0.6299046874046326,
+      "logits/rejected": -0.6121450662612915,
+      "logps/chosen": -0.8063814640045166,
+      "logps/rejected": -0.5979243516921997,
+      "loss": 0.6784,
+      "odds_ratio_loss": 6.7539262771606445,
+      "rewards/accuracies": 0.4296875,
+      "rewards/chosen": -0.05979243293404579,
+      "rewards/margins": 0.020845718681812286,
+      "rewards/rejected": -0.08063814043998718,
+      "sft_loss": 0.0030515664257109165,
+      "step": 147
+    },
+    {
+      "epoch": 0.8117929379499486,
+      "grad_norm": 3.6637986589772233,
+      "learning_rate": 5.178394771125969e-07,
+      "logits/chosen": -0.6278913021087646,
+      "logits/rejected": -0.5689703226089478,
+      "logps/chosen": -0.7562225461006165,
+      "logps/rejected": -0.6262752413749695,
+      "loss": 0.7032,
+      "odds_ratio_loss": 6.891101837158203,
+      "rewards/accuracies": 0.484375,
+      "rewards/chosen": -0.06262752413749695,
+      "rewards/margins": 0.012994730845093727,
+      "rewards/rejected": -0.07562224566936493,
+      "sft_loss": 0.01412445493042469,
+      "step": 148
+    },
+    {
+      "epoch": 0.8172780253685293,
+      "grad_norm": 2.3203742871591144,
+      "learning_rate": 4.888462296487129e-07,
+      "logits/chosen": -0.6456272602081299,
+      "logits/rejected": -0.6417357325553894,
+      "logps/chosen": -0.6021786332130432,
+      "logps/rejected": -0.6396222114562988,
+      "loss": 0.7282,
+      "odds_ratio_loss": 7.244186878204346,
+      "rewards/accuracies": 0.3359375,
+      "rewards/chosen": -0.06396222114562988,
+      "rewards/margins": -0.0037443540059030056,
+      "rewards/rejected": -0.06021786853671074,
+      "sft_loss": 0.0037422627210617065,
+      "step": 149
+    },
+    {
+      "epoch": 0.82276311278711,
+      "grad_norm": 2.5512980917433157,
+      "learning_rate": 4.6060004370984763e-07,
+      "logits/chosen": -0.631007969379425,
+      "logits/rejected": -0.5933969020843506,
+      "logps/chosen": -0.8098146915435791,
+      "logps/rejected": -0.6024014353752136,
+      "loss": 0.6766,
+      "odds_ratio_loss": 6.713679790496826,
+      "rewards/accuracies": 0.4921875,
+      "rewards/chosen": -0.06024014204740524,
+      "rewards/margins": 0.02074132300913334,
+      "rewards/rejected": -0.08098147064447403,
+      "sft_loss": 0.005258024670183659,
+      "step": 150
+    },
+    {
+      "epoch": 0.8282482002056908,
+      "grad_norm": 2.198032076864518,
+      "learning_rate": 4.331114116009938e-07,
+      "logits/chosen": -0.6112239360809326,
+      "logits/rejected": -0.55472332239151,
+      "logps/chosen": -0.7825672030448914,
+      "logps/rejected": -0.6584952473640442,
+      "loss": 0.7438,
+      "odds_ratio_loss": 7.401620864868164,
+      "rewards/accuracies": 0.3828125,
+      "rewards/chosen": -0.06584953516721725,
+      "rewards/margins": 0.012407200410962105,
+      "rewards/rejected": -0.07825673371553421,
+      "sft_loss": 0.0036755804903805256,
+      "step": 151
+    },
+    {
+      "epoch": 0.8337332876242715,
+      "grad_norm": 2.062470910623532,
+      "learning_rate": 4.063905442268201e-07,
+      "logits/chosen": -0.6306271553039551,
+      "logits/rejected": -0.6064797043800354,
+      "logps/chosen": -0.6712342500686646,
+      "logps/rejected": -0.6343151926994324,
+      "loss": 0.7212,
+      "odds_ratio_loss": 7.176795959472656,
+      "rewards/accuracies": 0.375,
+      "rewards/chosen": -0.063431516289711,
+      "rewards/margins": 0.0036919128615409136,
+      "rewards/rejected": -0.0671234279870987,
+      "sft_loss": 0.0035129080060869455,
+      "step": 152
+    },
+    {
+      "epoch": 0.8392183750428522,
+      "grad_norm": 2.3428435069292166,
+      "learning_rate": 3.80447367298738e-07,
+      "logits/chosen": -0.6870932579040527,
+      "logits/rejected": -0.6724913716316223,
+      "logps/chosen": -0.6094024777412415,
+      "logps/rejected": -0.6165374517440796,
+      "loss": 0.7051,
+      "odds_ratio_loss": 7.0048112869262695,
+      "rewards/accuracies": 0.3671875,
+      "rewards/chosen": -0.0616537444293499,
+      "rewards/margins": -0.0007134978659451008,
+      "rewards/rejected": -0.06094024330377579,
+      "sft_loss": 0.004614294972270727,
+      "step": 153
+    },
+    {
+      "epoch": 0.844703462461433,
+      "grad_norm": 2.162714758637227,
+      "learning_rate": 3.5529151764790715e-07,
+      "logits/chosen": -0.6563050746917725,
+      "logits/rejected": -0.6312206983566284,
+      "logps/chosen": -0.7330479025840759,
+      "logps/rejected": -0.6632793545722961,
+      "loss": 0.7478,
+      "odds_ratio_loss": 7.4362664222717285,
+      "rewards/accuracies": 0.359375,
+      "rewards/chosen": -0.06632794439792633,
+      "rewards/margins": 0.006976846605539322,
+      "rewards/rejected": -0.07330479472875595,
+      "sft_loss": 0.004126606043428183,
+      "step": 154
+    },
+    {
+      "epoch": 0.8501885498800137,
+      "grad_norm": 2.061116429438832,
+      "learning_rate": 3.3093233964554464e-07,
+      "logits/chosen": -0.6102815866470337,
+      "logits/rejected": -0.6046280860900879,
+      "logps/chosen": -0.6141259670257568,
+      "logps/rejected": -0.5823516845703125,
+      "loss": 0.6607,
+      "odds_ratio_loss": 6.578813076019287,
+      "rewards/accuracies": 0.4609375,
+      "rewards/chosen": -0.05823516845703125,
+      "rewards/margins": 0.0031774300150573254,
+      "rewards/rejected": -0.061412595212459564,
+      "sft_loss": 0.002771932166069746,
+      "step": 155
+    },
+    {
+      "epoch": 0.8556736372985945,
+      "grad_norm": 2.432594869696479,
+      "learning_rate": 3.0737888173187067e-07,
+      "logits/chosen": -0.6550750136375427,
+      "logits/rejected": -0.6305856108665466,
+      "logps/chosen": -0.6801073551177979,
+      "logps/rejected": -0.5993965268135071,
+      "loss": 0.6803,
+      "odds_ratio_loss": 6.759056568145752,
+      "rewards/accuracies": 0.4140625,
+      "rewards/chosen": -0.05993964523077011,
+      "rewards/margins": 0.008071082644164562,
+      "rewards/rejected": -0.06801073253154755,
+      "sft_loss": 0.004404113162308931,
+      "step": 156
+    },
+    {
+      "epoch": 0.8611587247171751,
+      "grad_norm": 2.0454259039650977,
+      "learning_rate": 2.8463989305498596e-07,
+      "logits/chosen": -0.5875197649002075,
+      "logits/rejected": -0.6273292303085327,
+      "logps/chosen": -0.5868790149688721,
+      "logps/rejected": -0.5973340272903442,
+      "loss": 0.6798,
+      "odds_ratio_loss": 6.763144493103027,
+      "rewards/accuracies": 0.375,
+      "rewards/chosen": -0.05973340570926666,
+      "rewards/margins": -0.0010455029550939798,
+      "rewards/rejected": -0.058687906712293625,
+      "sft_loss": 0.0034761279821395874,
+      "step": 157
+    },
+    {
+      "epoch": 0.8666438121357559,
+      "grad_norm": 2.158571117053065,
+      "learning_rate": 2.6272382022091704e-07,
+      "logits/chosen": -0.6387627124786377,
+      "logits/rejected": -0.6411675214767456,
+      "logps/chosen": -0.6698893308639526,
+      "logps/rejected": -0.5845687389373779,
+      "loss": 0.6596,
+      "odds_ratio_loss": 6.559039115905762,
+      "rewards/accuracies": 0.4765625,
+      "rewards/chosen": -0.05845687910914421,
+      "rewards/margins": 0.008532052859663963,
+      "rewards/rejected": -0.06698893010616302,
+      "sft_loss": 0.0036818967200815678,
+      "step": 158
+    },
+    {
+      "epoch": 0.8721288995543367,
+      "grad_norm": 2.1296893473193985,
+      "learning_rate": 2.4163880415604913e-07,
+      "logits/chosen": -0.5978600978851318,
+      "logits/rejected": -0.5640897154808044,
+      "logps/chosen": -0.6199032664299011,
+      "logps/rejected": -0.6052448749542236,
+      "loss": 0.6879,
+      "odds_ratio_loss": 6.839813232421875,
+      "rewards/accuracies": 0.40625,
+      "rewards/chosen": -0.06052448973059654,
+      "rewards/margins": 0.0014658391010016203,
+      "rewards/rejected": -0.06199032440781593,
+      "sft_loss": 0.003927412908524275,
+      "step": 159
+    },
+    {
+      "epoch": 0.8776139869729174,
+      "grad_norm": 2.155495858759351,
+      "learning_rate": 2.2139267708310457e-07,
+      "logits/chosen": -0.676996111869812,
+      "logits/rejected": -0.6428037285804749,
+      "logps/chosen": -0.6014530658721924,
+      "logps/rejected": -0.5981815457344055,
+      "loss": 0.6794,
+      "odds_ratio_loss": 6.754974365234375,
+      "rewards/accuracies": 0.4140625,
+      "rewards/chosen": -0.05981815233826637,
+      "rewards/margins": 0.00032715569250285625,
+      "rewards/rejected": -0.060145311057567596,
+      "sft_loss": 0.0038914589677006006,
+      "step": 160
+    },
+    {
+      "epoch": 0.8830990743914982,
+      "grad_norm": 2.6138265531338565,
+      "learning_rate": 2.0199295961178893e-07,
+      "logits/chosen": -0.6286705136299133,
+      "logits/rejected": -0.6172723770141602,
+      "logps/chosen": -0.6388348937034607,
+      "logps/rejected": -0.5947953462600708,
+      "loss": 0.6768,
+      "odds_ratio_loss": 6.710721492767334,
+      "rewards/accuracies": 0.4140625,
+      "rewards/chosen": -0.05947953462600708,
+      "rewards/margins": 0.004403956700116396,
+      "rewards/rejected": -0.06388349086046219,
+      "sft_loss": 0.005766674876213074,
+      "step": 161
+    },
+    {
+      "epoch": 0.8885841618100788,
+      "grad_norm": 2.022101568731076,
+      "learning_rate": 1.8344685794519507e-07,
+      "logits/chosen": -0.6218512058258057,
+      "logits/rejected": -0.5689311027526855,
+      "logps/chosen": -0.6789796352386475,
+      "logps/rejected": -0.6389855742454529,
+      "loss": 0.7205,
+      "odds_ratio_loss": 7.172222137451172,
+      "rewards/accuracies": 0.421875,
+      "rewards/chosen": -0.06389855593442917,
+      "rewards/margins": 0.003999411594122648,
+      "rewards/rejected": -0.06789796054363251,
+      "sft_loss": 0.0032456754706799984,
+      "step": 162
+    },
+    {
+      "epoch": 0.8940692492286596,
+      "grad_norm": 2.2581765266272185,
+      "learning_rate": 1.6576126120299046e-07,
+      "logits/chosen": -0.6435633301734924,
+      "logits/rejected": -0.589432954788208,
+      "logps/chosen": -0.7249476313591003,
+      "logps/rejected": -0.6334548592567444,
+      "loss": 0.7181,
+      "odds_ratio_loss": 7.1389479637146,
+      "rewards/accuracies": 0.3671875,
+      "rewards/chosen": -0.06334548443555832,
+      "rewards/margins": 0.009149276651442051,
+      "rewards/rejected": -0.0724947601556778,
+      "sft_loss": 0.004166773054748774,
+      "step": 163
+    },
+    {
+      "epoch": 0.8995543366472403,
+      "grad_norm": 2.2259746913759004,
+      "learning_rate": 1.4894273886239208e-07,
+      "logits/chosen": -0.6249025464057922,
+      "logits/rejected": -0.5969172716140747,
+      "logps/chosen": -0.633272647857666,
+      "logps/rejected": -0.6413824558258057,
+      "loss": 0.726,
+      "odds_ratio_loss": 7.205265045166016,
+      "rewards/accuracies": 0.3671875,
+      "rewards/chosen": -0.06413824111223221,
+      "rewards/margins": -0.0008109819609671831,
+      "rewards/rejected": -0.06332726776599884,
+      "sft_loss": 0.005517784971743822,
+      "step": 164
+    },
+    {
+      "epoch": 0.905039424065821,
+      "grad_norm": 2.2159816572748468,
+      "learning_rate": 1.3299753831787193e-07,
+      "logits/chosen": -0.601813018321991,
+      "logits/rejected": -0.6476290822029114,
+      "logps/chosen": -0.6103772521018982,
+      "logps/rejected": -0.6375135183334351,
+      "loss": 0.7213,
+      "odds_ratio_loss": 7.173505783081055,
+      "rewards/accuracies": 0.3828125,
+      "rewards/chosen": -0.06375134736299515,
+      "rewards/margins": -0.002713626716285944,
+      "rewards/rejected": -0.06103772297501564,
+      "sft_loss": 0.003954712767153978,
+      "step": 165
+    },
+    {
+      "epoch": 0.9105245114844018,
+      "grad_norm": 1.987492056492753,
+      "learning_rate": 1.1793158256050708e-07,
+      "logits/chosen": -0.628088116645813,
+      "logits/rejected": -0.591625452041626,
+      "logps/chosen": -0.6703177690505981,
+      "logps/rejected": -0.5828872323036194,
+      "loss": 0.6629,
+      "odds_ratio_loss": 6.595256805419922,
+      "rewards/accuracies": 0.4296875,
+      "rewards/chosen": -0.05828872323036194,
+      "rewards/margins": 0.008743051439523697,
+      "rewards/rejected": -0.06703177839517593,
+      "sft_loss": 0.003396927611902356,
+      "step": 166
+    },
+    {
+      "epoch": 0.9160095989029825,
+      "grad_norm": 2.059761656496877,
+      "learning_rate": 1.0375046797782868e-07,
+      "logits/chosen": -0.6672204732894897,
+      "logits/rejected": -0.6035845875740051,
+      "logps/chosen": -0.6069723963737488,
+      "logps/rejected": -0.6074884533882141,
+      "loss": 0.6911,
+      "odds_ratio_loss": 6.8784990310668945,
+      "rewards/accuracies": 0.3984375,
+      "rewards/chosen": -0.06074884906411171,
+      "rewards/margins": -5.1606097258627415e-05,
+      "rewards/rejected": -0.06069723516702652,
+      "sft_loss": 0.0032916096970438957,
+      "step": 167
+    },
+    {
+      "epoch": 0.9214946863215633,
+      "grad_norm": 1.8571804970978707,
+      "learning_rate": 9.045946227499298e-08,
+      "logits/chosen": -0.670376718044281,
+      "logits/rejected": -0.5822762846946716,
+      "logps/chosen": -0.7293166518211365,
+      "logps/rejected": -0.6415222883224487,
+      "loss": 0.726,
+      "odds_ratio_loss": 7.229472637176514,
+      "rewards/accuracies": 0.34375,
+      "rewards/chosen": -0.06415222585201263,
+      "rewards/margins": 0.008779437281191349,
+      "rewards/rejected": -0.072931669652462,
+      "sft_loss": 0.0030599033925682306,
+      "step": 168
+    },
+    {
+      "epoch": 0.926979773740144,
+      "grad_norm": 2.279667665781299,
+      "learning_rate": 7.806350251804484e-08,
+      "logits/chosen": -0.6420009732246399,
+      "logits/rejected": -0.5824066400527954,
+      "logps/chosen": -0.875396728515625,
+      "logps/rejected": -0.6270475387573242,
+      "loss": 0.7041,
+      "odds_ratio_loss": 7.000705242156982,
+      "rewards/accuracies": 0.46875,
+      "rewards/chosen": -0.06270475685596466,
+      "rewards/margins": 0.02483491040766239,
+      "rewards/rejected": -0.0875396728515625,
+      "sft_loss": 0.004043279215693474,
+      "step": 169
+    },
+    {
+      "epoch": 0.9324648611587247,
+      "grad_norm": 1.9899455007963414,
+      "learning_rate": 6.6567193299997e-08,
+      "logits/chosen": -0.6606361269950867,
+      "logits/rejected": -0.6265894174575806,
+      "logps/chosen": -0.8179676532745361,
+      "logps/rejected": -0.6608296036720276,
+      "loss": 0.7442,
+      "odds_ratio_loss": 7.407998085021973,
+      "rewards/accuracies": 0.34375,
+      "rewards/chosen": -0.06608295440673828,
+      "rewards/margins": 0.015713810920715332,
+      "rewards/rejected": -0.08179676532745361,
+      "sft_loss": 0.0034413619432598352,
+      "step": 170
+    },
+    {
+      "epoch": 0.9379499485773054,
+      "grad_norm": 2.432140341651671,
+      "learning_rate": 5.597480503041486e-08,
+      "logits/chosen": -0.6267792582511902,
+      "logits/rejected": -0.6068964004516602,
+      "logps/chosen": -0.6993312835693359,
+      "logps/rejected": -0.6519731283187866,
+      "loss": 0.7372,
+      "odds_ratio_loss": 7.303807258605957,
+      "rewards/accuracies": 0.40625,
+      "rewards/chosen": -0.06519731879234314,
+      "rewards/margins": 0.004735812544822693,
+      "rewards/rejected": -0.06993313133716583,
+      "sft_loss": 0.006868740543723106,
+      "step": 171
+    },
+    {
+      "epoch": 0.9434350359958862,
+      "grad_norm": 2.1615902439918186,
+      "learning_rate": 4.629027234912986e-08,
+      "logits/chosen": -0.662708044052124,
+      "logits/rejected": -0.6557428240776062,
+      "logps/chosen": -0.608113169670105,
+      "logps/rejected": -0.5925155878067017,
+      "loss": 0.6778,
+      "odds_ratio_loss": 6.741554260253906,
+      "rewards/accuracies": 0.390625,
+      "rewards/chosen": -0.059251561760902405,
+      "rewards/margins": 0.0015597562305629253,
+      "rewards/rejected": -0.06081131845712662,
+      "sft_loss": 0.003658043686300516,
+      "step": 172
+    },
+    {
+      "epoch": 0.948920123414467,
+      "grad_norm": 2.2861540937365947,
+      "learning_rate": 3.7517192664685844e-08,
+      "logits/chosen": -0.6392232179641724,
+      "logits/rejected": -0.6070412993431091,
+      "logps/chosen": -0.6354994177818298,
+      "logps/rejected": -0.628402590751648,
+      "loss": 0.7149,
+      "odds_ratio_loss": 7.101507186889648,
+      "rewards/accuracies": 0.359375,
+      "rewards/chosen": -0.06284026056528091,
+      "rewards/margins": 0.0007096900371834636,
+      "rewards/rejected": -0.0635499432682991,
+      "sft_loss": 0.004706856328994036,
+      "step": 173
+    },
+    {
+      "epoch": 0.9544052108330476,
+      "grad_norm": 2.3390784115932726,
+      "learning_rate": 2.9658824818044328e-08,
+      "logits/chosen": -0.6274293065071106,
+      "logits/rejected": -0.5854682326316833,
+      "logps/chosen": -0.7155969142913818,
+      "logps/rejected": -0.6342028975486755,
+      "loss": 0.7122,
+      "odds_ratio_loss": 7.073307514190674,
+      "rewards/accuracies": 0.453125,
+      "rewards/chosen": -0.06342029571533203,
+      "rewards/margins": 0.008139407262206078,
+      "rewards/rejected": -0.07155969738960266,
+      "sft_loss": 0.004918898921459913,
+      "step": 174
+    },
+    {
+      "epoch": 0.9598902982516284,
+      "grad_norm": 2.563514632765213,
+      "learning_rate": 2.2718087872060925e-08,
+      "logits/chosen": -0.6743865013122559,
+      "logits/rejected": -0.6293442845344543,
+      "logps/chosen": -0.7085027098655701,
+      "logps/rejected": -0.6935465931892395,
+      "loss": 0.7803,
+      "odds_ratio_loss": 7.746453285217285,
+      "rewards/accuracies": 0.375,
+      "rewards/chosen": -0.06935466080904007,
+      "rewards/margins": 0.0014956118538975716,
+      "rewards/rejected": -0.07085027545690536,
+      "sft_loss": 0.0056967539712786674,
+      "step": 175
+    },
+    {
+      "epoch": 0.9653753856702091,
+      "grad_norm": 2.056659344905833,
+      "learning_rate": 1.6697560027171543e-08,
+      "logits/chosen": -0.620927631855011,
+      "logits/rejected": -0.5821961760520935,
+      "logps/chosen": -0.6993247866630554,
+      "logps/rejected": -0.5995112657546997,
+      "loss": 0.6786,
+      "odds_ratio_loss": 6.743234634399414,
+      "rewards/accuracies": 0.4453125,
+      "rewards/chosen": -0.05995112285017967,
+      "rewards/margins": 0.00998135469853878,
+      "rewards/rejected": -0.0699324831366539,
+      "sft_loss": 0.004281484521925449,
+      "step": 176
+    },
+    {
+      "epoch": 0.9708604730887899,
+      "grad_norm": 2.133552946595605,
+      "learning_rate": 1.1599477663696845e-08,
+      "logits/chosen": -0.6138472557067871,
+      "logits/rejected": -0.6400952339172363,
+      "logps/chosen": -0.6972988843917847,
+      "logps/rejected": -0.6593939661979675,
+      "loss": 0.7433,
+      "odds_ratio_loss": 7.3952131271362305,
+      "rewards/accuracies": 0.3984375,
+      "rewards/chosen": -0.06593939661979675,
+      "rewards/margins": 0.0037904919590801,
+      "rewards/rejected": -0.06972989439964294,
+      "sft_loss": 0.00373737677000463,
+      "step": 177
+    },
+    {
+      "epoch": 0.9763455605073705,
+      "grad_norm": 2.256606648624031,
+      "learning_rate": 7.425734511117e-09,
+      "logits/chosen": -0.6247913241386414,
+      "logits/rejected": -0.6045972108840942,
+      "logps/chosen": -0.5932058691978455,
+      "logps/rejected": -0.6002853512763977,
+      "loss": 0.685,
+      "odds_ratio_loss": 6.808387756347656,
+      "rewards/accuracies": 0.3671875,
+      "rewards/chosen": -0.06002853438258171,
+      "rewards/margins": -0.00070795021019876,
+      "rewards/rejected": -0.059320587664842606,
+      "sft_loss": 0.004185628145933151,
+      "step": 178
+    },
+    {
+      "epoch": 0.9818306479259513,
+      "grad_norm": 2.370987455595276,
+      "learning_rate": 4.17788094463023e-09,
+      "logits/chosen": -0.6684907674789429,
+      "logits/rejected": -0.613418698310852,
+      "logps/chosen": -0.6800941824913025,
+      "logps/rejected": -0.5861297845840454,
+      "loss": 0.6704,
+      "odds_ratio_loss": 6.643950462341309,
+      "rewards/accuracies": 0.375,
+      "rewards/chosen": -0.05861297994852066,
+      "rewards/margins": 0.009396445006132126,
+      "rewards/rejected": -0.06800942122936249,
+      "sft_loss": 0.006054120138287544,
+      "step": 179
+    },
+    {
+      "epoch": 0.9873157353445321,
+      "grad_norm": 2.134555240071597,
+      "learning_rate": 1.857123409250705e-09,
+      "logits/chosen": -0.6588638424873352,
+      "logits/rejected": -0.6565414071083069,
+      "logps/chosen": -0.6550459265708923,
+      "logps/rejected": -0.6189512014389038,
+      "loss": 0.7047,
+      "odds_ratio_loss": 7.008230209350586,
+      "rewards/accuracies": 0.3671875,
+      "rewards/chosen": -0.06189511716365814,
+      "rewards/margins": 0.0036094679962843657,
+      "rewards/rejected": -0.06550458818674088,
+      "sft_loss": 0.003911715466529131,
+      "step": 180
+    },
+    {
+      "epoch": 0.9928008227631128,
+      "grad_norm": 2.189370604534784,
+      "learning_rate": 4.6432397166285e-10,
+      "logits/chosen": -0.6217331886291504,
+      "logits/rejected": -0.553974986076355,
+      "logps/chosen": -0.6876395344734192,
+      "logps/rejected": -0.5693202018737793,
+      "loss": 0.6418,
+      "odds_ratio_loss": 6.377911567687988,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": -0.05693202465772629,
+      "rewards/margins": 0.011831930838525295,
+      "rewards/rejected": -0.06876395642757416,
+      "sft_loss": 0.003973289392888546,
+      "step": 181
+    },
+    {
+      "epoch": 0.9982859101816935,
+      "grad_norm": 2.1620947430558077,
+      "learning_rate": 0.0,
+      "logits/chosen": -0.7046372890472412,
+      "logits/rejected": -0.6097927689552307,
+      "logps/chosen": -0.9193150401115417,
+      "logps/rejected": -0.5992479920387268,
+      "loss": 0.6706,
+      "odds_ratio_loss": 6.670595645904541,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": -0.05992480367422104,
+      "rewards/margins": 0.032006699591875076,
+      "rewards/rejected": -0.09193150699138641,
+      "sft_loss": 0.003579822601750493,
+      "step": 182
+    },
+    {
+      "epoch": 0.9982859101816935,
+      "step": 182,
+      "total_flos": 58779245903872.0,
+      "train_loss": 0.733595297559277,
+      "train_runtime": 13688.6483,
+      "train_samples_per_second": 1.704,
+      "train_steps_per_second": 0.013
+    }
+  ],
+  "logging_steps": 1,
+  "max_steps": 182,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 1,
+  "save_steps": 182,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": true
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 58779245903872.0,
+  "train_batch_size": 1,
+  "trial_name": null,
+  "trial_params": null
+}