{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 1.0,
  "eval_steps": 500,
  "global_step": 500,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.002,
      "grad_norm": 0.0,
      "kl": 0.0,
      "learning_rate": 2e-08,
      "loss": 0.0,
      "reward": 0.125,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.125,
      "step": 1
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.004,
      "grad_norm": 0.6894801259040833,
      "kl": 0.0,
      "learning_rate": 4e-08,
      "loss": 0.0,
      "reward": 0.5625,
      "reward_std": 0.0883883461356163,
      "rewards/accuracy_reward": 0.5,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.0625,
      "step": 2
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1845.75,
      "epoch": 0.006,
      "grad_norm": 0.0036829786840826273,
      "kl": 4.696846008300781e-05,
      "learning_rate": 6e-08,
      "loss": 0.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.0,
      "step": 3
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.008,
      "grad_norm": 0.0038558689411729574,
      "kl": 9.775161743164062e-05,
      "learning_rate": 8e-08,
      "loss": 0.0,
      "reward": 0.125,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.125,
      "step": 4
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.01,
      "grad_norm": 0.7485816478729248,
      "kl": 0.00010919570922851562,
      "learning_rate": 1e-07,
      "loss": 0.0,
      "reward": 0.0625,
      "reward_std": 0.0883883461356163,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.0625,
      "step": 5
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1985.5,
      "epoch": 0.012,
      "grad_norm": 0.7458791732788086,
      "kl": 7.796287536621094e-05,
      "learning_rate": 1.2e-07,
      "loss": -0.023,
      "reward": 0.0625,
      "reward_std": 0.0883883461356163,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.0625,
      "step": 6
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.014,
      "grad_norm": 0.003527791704982519,
      "kl": 6.628036499023438e-05,
      "learning_rate": 1.4e-07,
      "loss": 0.0,
      "reward": 0.125,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.125,
      "step": 7
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.016,
      "grad_norm": 0.8426324129104614,
      "kl": 7.104873657226562e-05,
      "learning_rate": 1.6e-07,
      "loss": 0.0,
      "reward": 0.0625,
      "reward_std": 0.0883883461356163,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.0625,
      "step": 8
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.018,
      "grad_norm": 0.7430940270423889,
      "kl": 9.5367431640625e-05,
      "learning_rate": 1.8e-07,
      "loss": 0.0,
      "reward": 0.1875,
      "reward_std": 0.0883883461356163,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.1875,
      "step": 9
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.02,
      "grad_norm": 0.003660305170342326,
      "kl": 5.626678466796875e-05,
      "learning_rate": 2e-07,
      "loss": 0.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.0,
      "step": 10
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.022,
      "grad_norm": 0.9972493648529053,
      "kl": 0.0001544952392578125,
      "learning_rate": 2.1999999999999998e-07,
      "loss": 0.0,
      "reward": 0.0625,
      "reward_std": 0.0883883461356163,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.0625,
      "step": 11
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.024,
      "grad_norm": 0.003556522075086832,
      "kl": 6.723403930664062e-05,
      "learning_rate": 2.4e-07,
      "loss": 0.0,
      "reward": 0.125,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.125,
      "step": 12
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1682.25,
      "epoch": 0.026,
      "grad_norm": 0.0050394581630826,
      "kl": 8.845329284667969e-05,
      "learning_rate": 2.6e-07,
      "loss": 0.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.0,
      "step": 13
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.028,
      "grad_norm": 0.004075606819242239,
      "kl": 0.0001049041748046875,
      "learning_rate": 2.8e-07,
      "loss": 0.0,
      "reward": 0.25,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.25,
      "step": 14
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.03,
      "grad_norm": 0.6532822251319885,
      "kl": 7.2479248046875e-05,
      "learning_rate": 3e-07,
      "loss": 0.0,
      "reward": 0.1875,
      "reward_std": 0.0883883461356163,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.1875,
      "step": 15
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.032,
      "grad_norm": 0.7762453556060791,
      "kl": 0.00011777877807617188,
      "learning_rate": 3.2e-07,
      "loss": 0.0,
      "reward": 0.1875,
      "reward_std": 0.0883883461356163,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.1875,
      "step": 16
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.034,
      "grad_norm": 0.7456417679786682,
      "kl": 8.58306884765625e-05,
      "learning_rate": 3.4000000000000003e-07,
      "loss": 0.0,
      "reward": 0.0625,
      "reward_std": 0.0883883461356163,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.0625,
      "step": 17
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1770.5,
      "epoch": 0.036,
      "grad_norm": 0.0034950117114931345,
      "kl": 3.5762786865234375e-05,
      "learning_rate": 3.6e-07,
      "loss": 0.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.0,
      "step": 18
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.038,
      "grad_norm": 0.003600472817197442,
      "kl": 6.866455078125e-05,
      "learning_rate": 3.7999999999999996e-07,
      "loss": 0.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.0,
      "step": 19
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1856.25,
      "epoch": 0.04,
      "grad_norm": 0.8460555076599121,
      "kl": 8.106231689453125e-05,
      "learning_rate": 4e-07,
      "loss": 0.0814,
      "reward": 0.0625,
      "reward_std": 0.0883883461356163,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.0625,
      "step": 20
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.042,
      "grad_norm": 0.0038911281153559685,
      "kl": 0.0001068115234375,
      "learning_rate": 4.1999999999999995e-07,
      "loss": 0.0,
      "reward": 0.25,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.25,
      "step": 21
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.044,
      "grad_norm": 0.7139887809753418,
      "kl": 0.0001125335693359375,
      "learning_rate": 4.3999999999999997e-07,
      "loss": 0.0,
      "reward": 0.1875,
      "reward_std": 0.0883883461356163,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.1875,
      "step": 22
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1855.25,
      "epoch": 0.046,
      "grad_norm": 0.005301118828356266,
      "kl": 5.6862831115722656e-05,
      "learning_rate": 4.6e-07,
      "loss": 0.0,
      "reward": 0.125,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.125,
      "step": 23
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.048,
      "grad_norm": 0.0036839963868260384,
      "kl": 7.62939453125e-05,
      "learning_rate": 4.8e-07,
      "loss": 0.0,
      "reward": 0.125,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.125,
      "step": 24
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.05,
      "grad_norm": 0.003639479400590062,
      "kl": 8.678436279296875e-05,
      "learning_rate": 5e-07,
      "loss": 0.0,
      "reward": 0.25,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.25,
      "step": 25
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.052,
      "grad_norm": 0.8071563243865967,
      "kl": 8.535385131835938e-05,
      "learning_rate": 5.2e-07,
      "loss": 0.0,
      "reward": 0.0625,
      "reward_std": 0.0883883461356163,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.0625,
      "step": 26
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.054,
      "grad_norm": 0.0037776094395667315,
      "kl": 0.00011014938354492188,
      "learning_rate": 5.4e-07,
      "loss": 0.0,
      "reward": 0.25,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.25,
      "step": 27
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1608.75,
      "epoch": 0.056,
      "grad_norm": 0.6625211834907532,
      "kl": 4.398822784423828e-05,
      "learning_rate": 5.6e-07,
      "loss": 0.0,
      "reward": 0.0625,
      "reward_std": 0.0883883461356163,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.0625,
      "step": 28
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1841.25,
      "epoch": 0.058,
      "grad_norm": 0.0036761562805622816,
      "kl": 8.678436279296875e-05,
      "learning_rate": 5.8e-07,
      "loss": 0.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.0,
      "step": 29
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.06,
      "grad_norm": 0.0033802345860749483,
      "kl": 5.7697296142578125e-05,
      "learning_rate": 6e-07,
      "loss": 0.0,
      "reward": 0.125,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.125,
      "step": 30
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.062,
      "grad_norm": 0.003240807680413127,
      "kl": 6.29425048828125e-05,
      "learning_rate": 6.2e-07,
      "loss": 0.0,
      "reward": 0.125,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.125,
      "step": 31
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.064,
      "grad_norm": 0.0036673492286354303,
      "kl": 7.152557373046875e-05,
      "learning_rate": 6.4e-07,
      "loss": 0.0,
      "reward": 0.125,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.125,
      "step": 32
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1735.75,
      "epoch": 0.066,
      "grad_norm": 0.8675172328948975,
      "kl": 2.682209014892578e-05,
      "learning_rate": 6.6e-07,
      "loss": -0.0631,
      "reward": 0.0625,
      "reward_std": 0.0883883461356163,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.0625,
      "step": 33
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.068,
      "grad_norm": 0.8184774518013,
      "kl": 9.775161743164062e-05,
      "learning_rate": 6.800000000000001e-07,
      "loss": 0.0,
      "reward": 0.0625,
      "reward_std": 0.0883883461356163,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.0625,
      "step": 34
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.07,
      "grad_norm": 0.003660478862002492,
      "kl": 0.00010251998901367188,
      "learning_rate": 7e-07,
      "loss": 0.0,
      "reward": 0.25,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.25,
      "step": 35
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.072,
      "grad_norm": 0.004034126177430153,
      "kl": 6.4849853515625e-05,
      "learning_rate": 7.2e-07,
      "loss": 0.0,
      "reward": 0.125,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.125,
      "step": 36
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.074,
      "grad_norm": 0.7589001655578613,
      "kl": 0.00010824203491210938,
      "learning_rate": 7.4e-07,
      "loss": 0.0,
      "reward": 0.0625,
      "reward_std": 0.0883883461356163,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.0625,
      "step": 37
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.076,
      "grad_norm": 0.6517212986946106,
      "kl": 8.20159912109375e-05,
      "learning_rate": 7.599999999999999e-07,
      "loss": 0.0,
      "reward": 0.0625,
      "reward_std": 0.0883883461356163,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.0625,
      "step": 38
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.078,
      "grad_norm": 0.8392653465270996,
      "kl": 0.0001430511474609375,
      "learning_rate": 7.799999999999999e-07,
      "loss": 0.0,
      "reward": 0.1875,
      "reward_std": 0.0883883461356163,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.1875,
      "step": 39
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.08,
      "grad_norm": 0.990591824054718,
      "kl": 0.0001125335693359375,
      "learning_rate": 8e-07,
      "loss": 0.0,
      "reward": 0.125,
      "reward_std": 0.1767766922712326,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.125,
      "step": 40
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.082,
      "grad_norm": 0.7600575685501099,
      "kl": 0.00014066696166992188,
      "learning_rate": 8.199999999999999e-07,
      "loss": 0.0,
      "reward": 0.0625,
      "reward_std": 0.0883883461356163,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.0625,
      "step": 41
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.084,
      "grad_norm": 0.004982170183211565,
      "kl": 0.00013589859008789062,
      "learning_rate": 8.399999999999999e-07,
      "loss": 0.0,
      "reward": 0.25,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.25,
      "step": 42
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.086,
      "grad_norm": 0.6588786840438843,
      "kl": 0.00019502639770507812,
      "learning_rate": 8.599999999999999e-07,
      "loss": 0.0,
      "reward": 0.1875,
      "reward_std": 0.0883883461356163,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.1875,
      "step": 43
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.088,
      "grad_norm": 0.876471996307373,
      "kl": 0.00020933151245117188,
      "learning_rate": 8.799999999999999e-07,
      "loss": 0.0,
      "reward": 0.0625,
      "reward_std": 0.0883883461356163,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.0625,
      "step": 44
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1857.0,
      "epoch": 0.09,
      "grad_norm": 0.007592031732201576,
      "kl": 0.00015783309936523438,
      "learning_rate": 9e-07,
      "loss": 0.0,
      "reward": 0.125,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.125,
      "step": 45
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.092,
      "grad_norm": 0.649493932723999,
      "kl": 0.00015401840209960938,
      "learning_rate": 9.2e-07,
      "loss": 0.0,
      "reward": 0.1875,
      "reward_std": 0.0883883461356163,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.1875,
      "step": 46
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.094,
      "grad_norm": 0.0060058352537453175,
      "kl": 0.00024318695068359375,
      "learning_rate": 9.399999999999999e-07,
      "loss": 0.0,
      "reward": 0.25,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.25,
      "step": 47
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1703.5,
      "epoch": 0.096,
      "grad_norm": 0.7367803454399109,
      "kl": 0.00028061866760253906,
      "learning_rate": 9.6e-07,
      "loss": 0.0,
      "reward": 0.0625,
      "reward_std": 0.0883883461356163,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.0625,
      "step": 48
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.098,
      "grad_norm": 0.7102321982383728,
      "kl": 0.0003223419189453125,
      "learning_rate": 9.8e-07,
      "loss": 0.0,
      "reward": 0.1875,
      "reward_std": 0.0883883461356163,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.1875,
      "step": 49
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.1,
      "grad_norm": 0.5775962471961975,
      "kl": 0.00010824203491210938,
      "learning_rate": 1e-06,
      "loss": 0.0,
      "reward": 0.1875,
      "reward_std": 0.0883883461356163,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.1875,
      "step": 50
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.102,
      "grad_norm": 0.0037765211891382933,
      "kl": 7.62939453125e-05,
      "learning_rate": 9.999890338174275e-07,
      "loss": 0.0,
      "reward": 0.125,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.125,
      "step": 51
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1991.5,
      "epoch": 0.104,
      "grad_norm": 0.8791236281394958,
      "kl": 0.00046634674072265625,
      "learning_rate": 9.999561358041868e-07,
      "loss": 0.0,
      "reward": 0.0625,
      "reward_std": 0.0883883461356163,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.0625,
      "step": 52
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.106,
      "grad_norm": 0.006120009813457727,
      "kl": 0.0004935264587402344,
      "learning_rate": 9.999013075636804e-07,
      "loss": 0.0,
      "reward": 0.125,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.125,
      "step": 53
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.108,
      "grad_norm": 0.006084068212658167,
      "kl": 0.0002951622009277344,
      "learning_rate": 9.998245517681593e-07,
      "loss": 0.0,
      "reward": 0.125,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.125,
      "step": 54
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1732.5,
      "epoch": 0.11,
      "grad_norm": 0.008206584490835667,
      "kl": 0.00015926361083984375,
      "learning_rate": 9.997258721585931e-07,
      "loss": 0.0,
      "reward": 0.5,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.5,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.0,
      "step": 55
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2031.5,
      "epoch": 0.112,
      "grad_norm": 0.06737767159938812,
      "kl": 0.0014257431030273438,
      "learning_rate": 9.996052735444862e-07,
      "loss": 0.0001,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.0,
      "step": 56
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.114,
      "grad_norm": 0.6304325461387634,
      "kl": 0.0003185272216796875,
      "learning_rate": 9.994627618036452e-07,
      "loss": 0.0,
      "reward": 0.1875,
      "reward_std": 0.0883883461356163,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.1875,
      "step": 57
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.116,
      "grad_norm": 0.6972232460975647,
      "kl": 0.000560760498046875,
      "learning_rate": 9.992983438818915e-07,
      "loss": 0.0,
      "reward": 0.0625,
      "reward_std": 0.0883883461356163,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.0625,
      "step": 58
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.118,
      "grad_norm": 0.026448842138051987,
      "kl": 0.0009822845458984375,
      "learning_rate": 9.991120277927223e-07,
      "loss": 0.0,
      "reward": 0.25,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.25,
      "step": 59
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.12,
      "grad_norm": 0.006011964753270149,
      "kl": 0.0007162094116210938,
      "learning_rate": 9.989038226169207e-07,
      "loss": 0.0,
      "reward": 0.75,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.5,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.25,
      "step": 60
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.122,
      "grad_norm": 0.6680987477302551,
      "kl": 0.0005612373352050781,
      "learning_rate": 9.98673738502114e-07,
      "loss": 0.0,
      "reward": 0.1875,
      "reward_std": 0.0883883461356163,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.1875,
      "step": 61
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1933.75,
      "epoch": 0.124,
      "grad_norm": 0.9241335988044739,
      "kl": 0.0006771087646484375,
      "learning_rate": 9.98421786662277e-07,
      "loss": -0.0443,
      "reward": 0.6875,
      "reward_std": 0.0883883461356163,
      "rewards/accuracy_reward": 0.5,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.1875,
      "step": 62
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.126,
      "grad_norm": 0.009985378012061119,
      "kl": 0.0005092620849609375,
      "learning_rate": 9.981479793771866e-07,
      "loss": 0.0,
      "reward": 0.125,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.125,
      "step": 63
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.128,
      "grad_norm": 0.03178563341498375,
      "kl": 0.0007648468017578125,
      "learning_rate": 9.97852329991824e-07,
      "loss": 0.0,
      "reward": 0.125,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.125,
      "step": 64
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1685.75,
      "epoch": 0.13,
      "grad_norm": 0.9064852595329285,
      "kl": 0.00341796875,
      "learning_rate": 9.975348529157229e-07,
      "loss": -0.0454,
      "reward": 0.0625,
      "reward_std": 0.0883883461356163,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.0625,
      "step": 65
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.132,
      "grad_norm": 0.7919860482215881,
      "kl": 0.001010894775390625,
      "learning_rate": 9.971955636222684e-07,
      "loss": 0.0,
      "reward": 0.1875,
      "reward_std": 0.0883883461356163,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.1875,
      "step": 66
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1887.25,
      "epoch": 0.134,
      "grad_norm": 0.8760141134262085,
      "kl": 0.000766754150390625,
      "learning_rate": 9.968344786479415e-07,
      "loss": 0.0659,
      "reward": 0.375,
      "reward_std": 0.3535533845424652,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.375,
      "step": 67
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.136,
      "grad_norm": 0.004815725143998861,
      "kl": 0.0005044937133789062,
      "learning_rate": 9.964516155915151e-07,
      "loss": 0.0,
      "reward": 0.25,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.25,
      "step": 68
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1797.25,
      "epoch": 0.138,
      "grad_norm": 0.8277981877326965,
      "kl": 0.0012836456298828125,
      "learning_rate": 9.960469931131936e-07,
      "loss": 0.1147,
      "reward": 0.9375,
      "reward_std": 0.2651650309562683,
      "rewards/accuracy_reward": 0.75,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.1875,
      "step": 69
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2000.25,
      "epoch": 0.14,
      "grad_norm": 0.8641435503959656,
      "kl": 0.0011048316955566406,
      "learning_rate": 9.956206309337066e-07,
      "loss": 0.0173,
      "reward": 0.25,
      "reward_std": 0.1767766922712326,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.25,
      "step": 70
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.142,
      "grad_norm": 0.009465116076171398,
      "kl": 0.00019884109497070312,
      "learning_rate": 9.951725498333448e-07,
      "loss": 0.0,
      "reward": 0.25,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.25,
      "step": 71
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1618.25,
      "epoch": 0.144,
      "grad_norm": 0.8773428797721863,
      "kl": 0.0106658935546875,
      "learning_rate": 9.947027716509488e-07,
      "loss": 0.0216,
      "reward": 0.6875,
      "reward_std": 0.0883883461356163,
      "rewards/accuracy_reward": 0.5,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.1875,
      "step": 72
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.146,
      "grad_norm": 0.0040911422111094,
      "kl": 0.0006923675537109375,
      "learning_rate": 9.942113192828444e-07,
      "loss": 0.0,
      "reward": 0.25,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.25,
      "step": 73
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.148,
      "grad_norm": 0.0059630973264575005,
      "kl": 0.000446319580078125,
      "learning_rate": 9.93698216681727e-07,
      "loss": 0.0,
      "reward": 0.75,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.5,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.25,
      "step": 74
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1836.75,
      "epoch": 0.15,
      "grad_norm": 1.146507978439331,
      "kl": 0.0004787445068359375,
      "learning_rate": 9.931634888554935e-07,
      "loss": -0.0918,
      "reward": 0.125,
      "reward_std": 0.1767766922712326,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.125,
      "step": 75
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.152,
      "grad_norm": 0.7911372184753418,
      "kl": 0.00030231475830078125,
      "learning_rate": 9.926071618660237e-07,
      "loss": 0.0,
      "reward": 0.1875,
      "reward_std": 0.0883883461356163,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.1875,
      "step": 76
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.154,
      "grad_norm": 0.8156778216362,
      "kl": 0.0006389617919921875,
      "learning_rate": 9.9202926282791e-07,
      "loss": 0.0,
      "reward": 0.1875,
      "reward_std": 0.0883883461356163,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.1875,
      "step": 77
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1915.25,
      "epoch": 0.156,
      "grad_norm": 0.005794746335595846,
      "kl": 0.0005588531494140625,
      "learning_rate": 9.91429819907136e-07,
      "loss": 0.0,
      "reward": 0.75,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.5,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.25,
      "step": 78
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1918.0,
      "epoch": 0.158,
      "grad_norm": 1.0309467315673828,
      "kl": 0.0005645751953125,
      "learning_rate": 9.908088623197048e-07,
      "loss": -0.0514,
      "reward": 0.125,
      "reward_std": 0.1767766922712326,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.125,
      "step": 79
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.16,
      "grad_norm": 0.9030879735946655,
      "kl": 0.0043182373046875,
      "learning_rate": 9.901664203302124e-07,
      "loss": 0.0002,
      "reward": 0.1875,
      "reward_std": 0.0883883461356163,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.1875,
      "step": 80
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.162,
      "grad_norm": 0.8063321113586426,
      "kl": 0.00075531005859375,
      "learning_rate": 9.895025252503755e-07,
      "loss": 0.0,
      "reward": 0.1875,
      "reward_std": 0.0883883461356163,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.1875,
      "step": 81
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.164,
      "grad_norm": 0.01648077741265297,
      "kl": 0.00039196014404296875,
      "learning_rate": 9.888172094375033e-07,
      "loss": 0.0,
      "reward": 0.25,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.25,
      "step": 82
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1855.0,
      "epoch": 0.166,
      "grad_norm": 0.7015650272369385,
      "kl": 0.0013475418090820312,
      "learning_rate": 9.881105062929221e-07,
      "loss": 0.0001,
      "reward": 0.0625,
      "reward_std": 0.0883883461356163,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.0625,
      "step": 83
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.168,
      "grad_norm": 0.006011520978063345,
      "kl": 0.00021600723266601562,
      "learning_rate": 9.873824502603459e-07,
      "loss": 0.0,
      "reward": 0.25,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.25,
      "step": 84
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1855.5,
      "epoch": 0.17,
      "grad_norm": 0.9796985983848572,
      "kl": 0.00054168701171875,
      "learning_rate": 9.866330768241983e-07,
      "loss": 0.0819,
      "reward": 0.5,
      "reward_std": 0.3535533845424652,
      "rewards/accuracy_reward": 0.25,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.25,
      "step": 85
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.172,
      "grad_norm": 0.006532273255288601,
      "kl": 0.0004496574401855469,
      "learning_rate": 9.85862422507884e-07,
      "loss": 0.0,
      "reward": 0.25,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.25,
      "step": 86
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.174,
      "grad_norm": 0.007916338741779327,
      "kl": 0.0005373954772949219,
      "learning_rate": 9.850705248720068e-07,
      "loss": 0.0,
      "reward": 0.25,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.25,
      "step": 87
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.176,
      "grad_norm": 0.006668766029179096,
      "kl": 0.00024175643920898438,
      "learning_rate": 9.8425742251254e-07,
      "loss": 0.0,
      "reward": 0.25,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.25,
      "step": 88
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1533.5,
      "epoch": 0.178,
      "grad_norm": 0.01681283488869667,
      "kl": 0.0006351470947265625,
      "learning_rate": 9.83423155058946e-07,
      "loss": 0.0,
      "reward": 0.125,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.125,
      "step": 89
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.18,
      "grad_norm": 0.6811097860336304,
      "kl": 0.0013413429260253906,
      "learning_rate": 9.825677631722435e-07,
      "loss": 0.0001,
      "reward": 0.1875,
      "reward_std": 0.0883883461356163,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.1875,
      "step": 90
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.182,
      "grad_norm": 0.7697494626045227,
      "kl": 0.0030584335327148438,
      "learning_rate": 9.816912885430258e-07,
      "loss": 0.0001,
      "reward": 0.1875,
      "reward_std": 0.0883883461356163,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.1875,
      "step": 91
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1913.25,
      "epoch": 0.184,
      "grad_norm": 1.1320937871932983,
      "kl": 0.001331329345703125,
      "learning_rate": 9.807937738894303e-07,
      "loss": 0.0536,
      "reward": 0.4375,
      "reward_std": 0.4419417306780815,
      "rewards/accuracy_reward": 0.25,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.1875,
      "step": 92
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1742.75,
      "epoch": 0.186,
      "grad_norm": 0.6808719635009766,
      "kl": 0.0010509490966796875,
      "learning_rate": 9.798752629550546e-07,
      "loss": -0.1501,
      "reward": 0.3125,
      "reward_std": 0.2651650309562683,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3125,
      "step": 93
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.188,
      "grad_norm": 0.007471662946045399,
      "kl": 0.0004634857177734375,
      "learning_rate": 9.78935800506826e-07,
      "loss": 0.0,
      "reward": 0.25,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.25,
      "step": 94
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.19,
      "grad_norm": 0.010327517054975033,
      "kl": 0.000568389892578125,
      "learning_rate": 9.779754323328192e-07,
      "loss": 0.0,
      "reward": 0.25,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.25,
      "step": 95
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.192,
      "grad_norm": 0.6696183681488037,
      "kl": 0.0009899139404296875,
      "learning_rate": 9.769942052400235e-07,
      "loss": 0.0,
      "reward": 0.1875,
      "reward_std": 0.0883883461356163,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.1875,
      "step": 96
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.194,
      "grad_norm": 0.8246662020683289,
      "kl": 0.0017538070678710938,
      "learning_rate": 9.759921670520634e-07,
      "loss": 0.0001,
      "reward": 0.1875,
      "reward_std": 0.0883883461356163,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.1875,
      "step": 97
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1928.5,
      "epoch": 0.196,
      "grad_norm": 0.955489456653595,
      "kl": 0.0009098052978515625,
      "learning_rate": 9.749693666068663e-07,
      "loss": 0.0467,
      "reward": 0.375,
      "reward_std": 0.3535533770918846,
      "rewards/accuracy_reward": 0.25,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.125,
      "step": 98
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.198,
      "grad_norm": 0.03343038633465767,
      "kl": 0.0007419586181640625,
      "learning_rate": 9.739258537542835e-07,
      "loss": 0.0,
      "reward": 0.125,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.125,
      "step": 99
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.2,
      "grad_norm": 0.006429341156035662,
      "kl": 0.001178741455078125,
      "learning_rate": 9.728616793536587e-07,
      "loss": 0.0,
      "reward": 0.25,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.25,
      "step": 100
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.202,
      "grad_norm": 0.00873623974621296,
      "kl": 0.0005130767822265625,
      "learning_rate": 9.717768952713511e-07,
      "loss": 0.0,
      "reward": 0.25,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.25,
      "step": 101
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1959.0,
      "epoch": 0.204,
      "grad_norm": 0.017306441441178322,
      "kl": 0.000946044921875,
      "learning_rate": 9.706715543782064e-07,
      "loss": 0.0,
      "reward": 0.125,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.125,
      "step": 102
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.206,
      "grad_norm": 0.7247556447982788,
      "kl": 0.0008764266967773438,
      "learning_rate": 9.695457105469804e-07,
      "loss": 0.0,
      "reward": 0.1875,
      "reward_std": 0.0883883461356163,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.1875,
      "step": 103
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.208,
      "grad_norm": 0.008871670812368393,
      "kl": 0.00033664703369140625,
      "learning_rate": 9.683994186497132e-07,
      "loss": 0.0,
      "reward": 0.75,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.5,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.25,
      "step": 104
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.21,
      "grad_norm": 0.007749219890683889,
      "kl": 0.00040531158447265625,
      "learning_rate": 9.672327345550543e-07,
      "loss": 0.0,
      "reward": 0.75,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.5,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.25,
      "step": 105
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.212,
      "grad_norm": 0.010708320885896683,
      "kl": 0.0010166168212890625,
      "learning_rate": 9.66045715125541e-07,
      "loss": 0.0,
      "reward": 0.25,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.25,
      "step": 106
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.214,
      "grad_norm": 0.007295151706784964,
      "kl": 0.0003333091735839844,
      "learning_rate": 9.648384182148252e-07,
      "loss": 0.0,
      "reward": 0.25,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.25,
      "step": 107
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.216,
      "grad_norm": 0.7922310829162598,
      "kl": 0.000408172607421875,
      "learning_rate": 9.636109026648554e-07,
      "loss": 0.0,
      "reward": 0.1875,
      "reward_std": 0.0883883461356163,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.1875,
      "step": 108
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.218,
      "grad_norm": 0.007899758405983448,
      "kl": 0.0006399154663085938,
      "learning_rate": 9.623632283030077e-07,
      "loss": 0.0,
      "reward": 0.25,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.25,
      "step": 109
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.22,
      "grad_norm": 0.9048980474472046,
      "kl": 0.001056671142578125,
      "learning_rate": 9.610954559391704e-07,
      "loss": 0.0,
      "reward": 0.6875,
      "reward_std": 0.0883883461356163,
      "rewards/accuracy_reward": 0.5,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.1875,
      "step": 110
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.222,
      "grad_norm": 0.010189698077738285,
      "kl": 0.00031948089599609375,
      "learning_rate": 9.598076473627796e-07,
      "loss": 0.0,
      "reward": 0.25,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.25,
      "step": 111
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1794.5,
      "epoch": 0.224,
      "grad_norm": 1.0683528184890747,
      "kl": 0.0034933090209960938,
      "learning_rate": 9.58499865339809e-07,
      "loss": -0.1162,
      "reward": 0.6875,
      "reward_std": 0.0883883461356163,
      "rewards/accuracy_reward": 0.5,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.1875,
      "step": 112
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.226,
      "grad_norm": 0.011858138255774975,
      "kl": 0.000347137451171875,
      "learning_rate": 9.571721736097088e-07,
      "loss": 0.0,
      "reward": 0.25,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.25,
      "step": 113
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.228,
      "grad_norm": 0.012356019578874111,
      "kl": 0.000885009765625,
      "learning_rate": 9.55824636882301e-07,
      "loss": 0.0,
      "reward": 0.25,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.25,
      "step": 114
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1752.5,
      "epoch": 0.23,
      "grad_norm": 1.0531798601150513,
      "kl": 0.001102447509765625,
      "learning_rate": 9.54457320834625e-07,
      "loss": 0.1434,
      "reward": 0.1875,
      "reward_std": 0.2651650309562683,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.1875,
      "step": 115
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.232,
      "grad_norm": 0.012715999968349934,
      "kl": 0.00096893310546875,
      "learning_rate": 9.530702921077358e-07,
      "loss": 0.0,
      "reward": 0.25,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.25,
      "step": 116
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2033.5,
      "epoch": 0.234,
      "grad_norm": 0.7506431937217712,
      "kl": 0.0050945281982421875,
      "learning_rate": 9.516636183034564e-07,
      "loss": 0.0002,
      "reward": 0.0625,
      "reward_std": 0.0883883461356163,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.0625,
      "step": 117
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.236,
      "grad_norm": 0.020277904346585274,
      "kl": 0.00078582763671875,
      "learning_rate": 9.502373679810839e-07,
      "loss": 0.0,
      "reward": 0.125,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.125,
      "step": 118
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1624.5,
      "epoch": 0.238,
      "grad_norm": 0.9856612086296082,
      "kl": 0.0022125244140625,
      "learning_rate": 9.487916106540465e-07,
      "loss": -0.0067,
      "reward": 0.1875,
      "reward_std": 0.0883883461356163,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.1875,
      "step": 119
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.24,
      "grad_norm": 0.007632279768586159,
      "kl": 0.000583648681640625,
      "learning_rate": 9.473264167865171e-07,
      "loss": 0.0,
      "reward": 0.125,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.125,
      "step": 120
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.242,
      "grad_norm": 0.007833893410861492,
      "kl": 0.0008401870727539062,
      "learning_rate": 9.458418577899774e-07,
      "loss": 0.0,
      "reward": 0.25,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.25,
      "step": 121
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.244,
      "grad_norm": 0.00713867275044322,
      "kl": 0.0011892318725585938,
      "learning_rate": 9.443380060197385e-07,
      "loss": 0.0,
      "reward": 0.125,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.125,
      "step": 122
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.246,
      "grad_norm": 0.011064039543271065,
      "kl": 0.0003185272216796875,
      "learning_rate": 9.428149347714143e-07,
      "loss": 0.0,
      "reward": 0.25,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.25,
      "step": 123
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.248,
      "grad_norm": 0.009095462039113045,
      "kl": 0.0006237030029296875,
      "learning_rate": 9.412727182773486e-07,
      "loss": 0.0,
      "reward": 0.25,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.25,
      "step": 124
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.25,
      "grad_norm": 0.007876625284552574,
      "kl": 0.00144195556640625,
      "learning_rate": 9.397114317029974e-07,
      "loss": 0.0001,
      "reward": 0.25,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.25,
      "step": 125
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.252,
      "grad_norm": 0.05358020216226578,
      "kl": 0.0010623931884765625,
      "learning_rate": 9.381311511432658e-07,
      "loss": 0.0,
      "reward": 0.625,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.5,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.125,
      "step": 126
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.254,
      "grad_norm": 0.030433854088187218,
      "kl": 0.0003848075866699219,
      "learning_rate": 9.36531953618799e-07,
      "loss": 0.0,
      "reward": 0.125,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.125,
      "step": 127
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.256,
      "grad_norm": 0.852528989315033,
      "kl": 0.0011425018310546875,
      "learning_rate": 9.34913917072228e-07,
      "loss": 0.0,
      "reward": 0.1875,
      "reward_std": 0.0883883461356163,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.1875,
      "step": 128
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.258,
      "grad_norm": 0.013770471327006817,
      "kl": 0.0007495880126953125,
      "learning_rate": 9.332771203643714e-07,
      "loss": 0.0,
      "reward": 0.75,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.5,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.25,
      "step": 129
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.26,
      "grad_norm": 0.7055062055587769,
      "kl": 0.0012664794921875,
      "learning_rate": 9.316216432703916e-07,
      "loss": 0.0001,
      "reward": 0.1875,
      "reward_std": 0.0883883461356163,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.1875,
      "step": 130
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.262,
      "grad_norm": 0.7149041295051575,
      "kl": 0.002803802490234375,
      "learning_rate": 9.299475664759068e-07,
      "loss": 0.0001,
      "reward": 0.0625,
      "reward_std": 0.0883883461356163,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.0625,
      "step": 131
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1480.25,
      "epoch": 0.264,
      "grad_norm": 0.014864159747958183,
      "kl": 0.031280517578125,
      "learning_rate": 9.282549715730579e-07,
      "loss": 0.0001,
      "reward": 0.125,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.125,
      "step": 132
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.266,
      "grad_norm": 0.8071376085281372,
      "kl": 0.00067901611328125,
      "learning_rate": 9.265439410565328e-07,
      "loss": 0.0,
      "reward": 0.1875,
      "reward_std": 0.0883883461356163,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.1875,
      "step": 133
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2047.25,
      "epoch": 0.268,
      "grad_norm": 0.009836402721703053,
      "kl": 0.0012359619140625,
      "learning_rate": 9.248145583195447e-07,
      "loss": 0.0,
      "reward": 0.25,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.25,
      "step": 134
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2012.25,
      "epoch": 0.27,
      "grad_norm": 0.007896827533841133,
      "kl": 0.0009002685546875,
      "learning_rate": 9.230669076497687e-07,
      "loss": 0.0,
      "reward": 0.25,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.25,
      "step": 135
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.272,
      "grad_norm": 0.0065347570925951,
      "kl": 0.000980377197265625,
      "learning_rate": 9.213010742252327e-07,
      "loss": 0.0,
      "reward": 0.25,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.25,
      "step": 136
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.274,
      "grad_norm": 0.8659334778785706,
      "kl": 0.00147247314453125,
      "learning_rate": 9.195171441101668e-07,
      "loss": 0.0001,
      "reward": 0.1875,
      "reward_std": 0.0883883461356163,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.1875,
      "step": 137
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.276,
      "grad_norm": 0.009184672497212887,
      "kl": 0.001373291015625,
      "learning_rate": 9.177152042508077e-07,
      "loss": 0.0001,
      "reward": 0.25,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.25,
      "step": 138
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.278,
      "grad_norm": 0.00881196279078722,
      "kl": 0.001476287841796875,
      "learning_rate": 9.158953424711624e-07,
      "loss": 0.0001,
      "reward": 0.25,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.25,
      "step": 139
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1876.0,
      "epoch": 0.28,
      "grad_norm": 0.7441470623016357,
      "kl": 0.0006885528564453125,
      "learning_rate": 9.140576474687263e-07,
      "loss": 0.06,
      "reward": 0.3125,
      "reward_std": 0.0883883461356163,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3125,
      "step": 140
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.282,
      "grad_norm": 0.008356408216059208,
      "kl": 0.00101470947265625,
      "learning_rate": 9.122022088101613e-07,
      "loss": 0.0,
      "reward": 0.25,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.25,
      "step": 141
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1991.5,
      "epoch": 0.284,
      "grad_norm": 0.9580811262130737,
      "kl": 0.006805419921875,
      "learning_rate": 9.103291169269299e-07,
      "loss": 0.0003,
      "reward": 0.5625,
      "reward_std": 0.0883883461356163,
      "rewards/accuracy_reward": 0.5,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.0625,
      "step": 142
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.286,
      "grad_norm": 0.008105803281068802,
      "kl": 0.002086639404296875,
      "learning_rate": 9.084384631108882e-07,
      "loss": 0.0001,
      "reward": 0.25,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.25,
      "step": 143
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.288,
      "grad_norm": 0.007048532832413912,
      "kl": 0.001430511474609375,
      "learning_rate": 9.065303395098358e-07,
      "loss": 0.0001,
      "reward": 0.25,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.25,
      "step": 144
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1714.5,
      "epoch": 0.29,
      "grad_norm": 0.881592869758606,
      "kl": 0.001071929931640625,
      "learning_rate": 9.046048391230247e-07,
      "loss": 0.1707,
      "reward": 0.25,
      "reward_std": 0.1767766922712326,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.25,
      "step": 145
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.292,
      "grad_norm": 0.008095495402812958,
      "kl": 0.00165557861328125,
      "learning_rate": 9.026620557966279e-07,
      "loss": 0.0001,
      "reward": 0.25,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.25,
      "step": 146
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.294,
      "grad_norm": 0.019286898896098137,
      "kl": 0.00104522705078125,
      "learning_rate": 9.007020842191634e-07,
      "loss": 0.0,
      "reward": 0.25,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.25,
      "step": 147
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.296,
      "grad_norm": 0.007973677478730679,
      "kl": 0.0015087127685546875,
      "learning_rate": 8.987250199168808e-07,
      "loss": 0.0001,
      "reward": 0.25,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.25,
      "step": 148
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.298,
      "grad_norm": 0.007949120365083218,
      "kl": 0.0004978179931640625,
      "learning_rate": 8.967309592491052e-07,
      "loss": 0.0,
      "reward": 0.125,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.125,
      "step": 149
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.3,
      "grad_norm": 0.007726718205958605,
      "kl": 0.001689910888671875,
      "learning_rate": 8.9471999940354e-07,
      "loss": 0.0001,
      "reward": 0.25,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.25,
      "step": 150
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.302,
      "grad_norm": 0.00826709158718586,
      "kl": 0.0013580322265625,
      "learning_rate": 8.926922383915315e-07,
      "loss": 0.0001,
      "reward": 0.25,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.25,
      "step": 151
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.304,
      "grad_norm": 0.007963276468217373,
      "kl": 0.00203704833984375,
      "learning_rate": 8.906477750432903e-07,
      "loss": 0.0001,
      "reward": 0.75,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.5,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.25,
      "step": 152
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.306,
      "grad_norm": 0.008207273669540882,
      "kl": 0.0008563995361328125,
      "learning_rate": 8.88586709003076e-07,
      "loss": 0.0,
      "reward": 0.25,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.25,
      "step": 153
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.308,
      "grad_norm": 0.010204545222222805,
      "kl": 0.0006160736083984375,
      "learning_rate": 8.865091407243394e-07,
      "loss": 0.0,
      "reward": 0.25,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.25,
      "step": 154
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.31,
      "grad_norm": 0.00880539882928133,
      "kl": 0.001453399658203125,
      "learning_rate": 8.844151714648274e-07,
      "loss": 0.0001,
      "reward": 0.25,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.25,
      "step": 155
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.312,
      "grad_norm": 0.010449129156768322,
      "kl": 0.0010318756103515625,
      "learning_rate": 8.823049032816478e-07,
      "loss": 0.0,
      "reward": 0.25,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.25,
      "step": 156
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.314,
      "grad_norm": 0.010188284330070019,
      "kl": 0.000789642333984375,
      "learning_rate": 8.801784390262943e-07,
      "loss": 0.0,
      "reward": 0.75,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.5,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.25,
      "step": 157
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1963.5,
      "epoch": 0.316,
      "grad_norm": 0.7694103717803955,
      "kl": 0.000640869140625,
      "learning_rate": 8.780358823396352e-07,
      "loss": 0.0318,
      "reward": 0.0625,
      "reward_std": 0.0883883461356163,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.0625,
      "step": 158
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1807.0,
      "epoch": 0.318,
      "grad_norm": 0.8925904631614685,
      "kl": 0.001445770263671875,
      "learning_rate": 8.758773376468604e-07,
      "loss": -0.1087,
      "reward": 0.1875,
      "reward_std": 0.0883883461356163,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.1875,
      "step": 159
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.32,
      "grad_norm": 0.7490972280502319,
      "kl": 0.01116180419921875,
      "learning_rate": 8.737029101523929e-07,
      "loss": 0.0004,
      "reward": 0.1875,
      "reward_std": 0.0883883461356163,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.1875,
      "step": 160
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.322,
      "grad_norm": 0.008408155292272568,
      "kl": 0.0014629364013671875,
      "learning_rate": 8.715127058347614e-07,
      "loss": 0.0001,
      "reward": 0.25,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.25,
      "step": 161
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.324,
      "grad_norm": 0.8539191484451294,
      "kl": 0.0008831024169921875,
      "learning_rate": 8.693068314414344e-07,
      "loss": 0.0,
      "reward": 0.1875,
      "reward_std": 0.0883883461356163,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.1875,
      "step": 162
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.326,
      "grad_norm": 0.009831397794187069,
      "kl": 0.000682830810546875,
      "learning_rate": 8.670853944836176e-07,
      "loss": 0.0,
      "reward": 0.25,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.25,
      "step": 163
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.328,
      "grad_norm": 0.011773304082453251,
      "kl": 0.001129150390625,
      "learning_rate": 8.648485032310144e-07,
      "loss": 0.0,
      "reward": 0.25,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.25,
      "step": 164
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.33,
      "grad_norm": 0.7276328206062317,
      "kl": 0.001377105712890625,
      "learning_rate": 8.625962667065487e-07,
      "loss": 0.0001,
      "reward": 0.1875,
      "reward_std": 0.0883883461356163,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.1875,
      "step": 165
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1512.75,
      "epoch": 0.332,
      "grad_norm": 0.008593901991844177,
      "kl": 0.0006933212280273438,
      "learning_rate": 8.603287946810513e-07,
      "loss": 0.0,
      "reward": 0.375,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.375,
      "step": 166
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.334,
      "grad_norm": 0.011681273579597473,
      "kl": 0.000759124755859375,
      "learning_rate": 8.580461976679099e-07,
      "loss": 0.0,
      "reward": 0.125,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.125,
      "step": 167
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.336,
      "grad_norm": 0.9719880819320679,
      "kl": 0.00112152099609375,
      "learning_rate": 8.557485869176825e-07,
      "loss": 0.0,
      "reward": 0.1875,
      "reward_std": 0.0883883461356163,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.1875,
      "step": 168
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1850.75,
      "epoch": 0.338,
      "grad_norm": 0.017566794529557228,
      "kl": 0.002292633056640625,
      "learning_rate": 8.534360744126753e-07,
      "loss": 0.0001,
      "reward": 0.25,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.25,
      "step": 169
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2016.0,
      "epoch": 0.34,
      "grad_norm": 1.1285712718963623,
      "kl": 0.00118255615234375,
      "learning_rate": 8.511087728614862e-07,
      "loss": 0.0114,
      "reward": 0.3125,
      "reward_std": 0.2651650384068489,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3125,
      "step": 170
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.342,
      "grad_norm": 0.009710059501230717,
      "kl": 0.001407623291015625,
      "learning_rate": 8.487667956935087e-07,
      "loss": 0.0001,
      "reward": 0.75,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.5,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.25,
      "step": 171
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.344,
      "grad_norm": 0.009776381775736809,
      "kl": 0.0014629364013671875,
      "learning_rate": 8.464102570534061e-07,
      "loss": 0.0001,
      "reward": 0.25,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.25,
      "step": 172
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.346,
      "grad_norm": 0.00871388241648674,
      "kl": 0.000507354736328125,
      "learning_rate": 8.440392717955475e-07,
      "loss": 0.0,
      "reward": 0.25,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.25,
      "step": 173
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.348,
      "grad_norm": 0.00912429578602314,
      "kl": 0.000762939453125,
      "learning_rate": 8.416539554784089e-07,
      "loss": 0.0,
      "reward": 0.25,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.25,
      "step": 174
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.35,
      "grad_norm": 0.011816666461527348,
      "kl": 0.0008449554443359375,
      "learning_rate": 8.392544243589427e-07,
      "loss": 0.0,
      "reward": 0.25,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.25,
      "step": 175
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.352,
      "grad_norm": 0.00985003262758255,
      "kl": 0.001529693603515625,
      "learning_rate": 8.368407953869103e-07,
      "loss": 0.0001,
      "reward": 0.125,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.125,
      "step": 176
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.354,
      "grad_norm": 0.00919476430863142,
      "kl": 0.00167083740234375,
      "learning_rate": 8.344131861991828e-07,
      "loss": 0.0001,
      "reward": 0.25,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.25,
      "step": 177
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.356,
      "grad_norm": 0.01162977609783411,
      "kl": 0.00091552734375,
      "learning_rate": 8.319717151140072e-07,
      "loss": 0.0,
      "reward": 0.25,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.25,
      "step": 178
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1970.5,
      "epoch": 0.358,
      "grad_norm": 0.01332745049148798,
      "kl": 0.001888275146484375,
      "learning_rate": 8.295165011252396e-07,
      "loss": 0.0001,
      "reward": 0.25,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.25,
      "step": 179
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1730.75,
      "epoch": 0.36,
      "grad_norm": 0.013342260383069515,
      "kl": 0.00089263916015625,
      "learning_rate": 8.270476638965461e-07,
      "loss": 0.0,
      "reward": 0.125,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.125,
      "step": 180
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.362,
      "grad_norm": 0.012930563651025295,
      "kl": 0.001483917236328125,
      "learning_rate": 8.245653237555705e-07,
      "loss": 0.0001,
      "reward": 0.625,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.5,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.125,
      "step": 181
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.364,
      "grad_norm": 0.03247583284974098,
      "kl": 0.00113677978515625,
      "learning_rate": 8.220696016880687e-07,
      "loss": 0.0,
      "reward": 0.25,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.25,
      "step": 182
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.366,
      "grad_norm": 0.8804360032081604,
      "kl": 0.0019130706787109375,
      "learning_rate": 8.195606193320136e-07,
      "loss": 0.0001,
      "reward": 0.1875,
      "reward_std": 0.0883883461356163,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.1875,
      "step": 183
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.368,
      "grad_norm": 0.007631482556462288,
      "kl": 0.001468658447265625,
      "learning_rate": 8.170384989716657e-07,
      "loss": 0.0001,
      "reward": 0.25,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.25,
      "step": 184
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.37,
      "grad_norm": 0.008286840282380581,
      "kl": 0.001613616943359375,
      "learning_rate": 8.145033635316128e-07,
      "loss": 0.0001,
      "reward": 0.25,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.25,
      "step": 185
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.372,
      "grad_norm": 0.016586236655712128,
      "kl": 0.0007076263427734375,
      "learning_rate": 8.119553365707802e-07,
      "loss": 0.0,
      "reward": 0.25,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.25,
      "step": 186
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.374,
      "grad_norm": 0.010034182108938694,
      "kl": 0.0008754730224609375,
      "learning_rate": 8.093945422764069e-07,
      "loss": 0.0,
      "reward": 0.25,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.25,
      "step": 187
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.376,
      "grad_norm": 0.7020014524459839,
      "kl": 0.0012836456298828125,
      "learning_rate": 8.068211054579943e-07,
      "loss": 0.0001,
      "reward": 0.1875,
      "reward_std": 0.0883883461356163,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.1875,
      "step": 188
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.378,
      "grad_norm": 1.163500189781189,
      "kl": 0.0016021728515625,
      "learning_rate": 8.04235151541222e-07,
      "loss": 0.0001,
      "reward": 0.125,
      "reward_std": 0.1767766922712326,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.125,
      "step": 189
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.38,
      "grad_norm": 0.8533800840377808,
      "kl": 0.0010223388671875,
      "learning_rate": 8.01636806561836e-07,
      "loss": 0.0,
      "reward": 0.1875,
      "reward_std": 0.0883883461356163,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.1875,
      "step": 190
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.382,
      "grad_norm": 0.8227788805961609,
      "kl": 0.0030975341796875,
      "learning_rate": 7.990261971595048e-07,
      "loss": 0.0001,
      "reward": 0.1875,
      "reward_std": 0.0883883461356163,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.1875,
      "step": 191
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.384,
      "grad_norm": 0.011001263745129108,
      "kl": 0.0008344650268554688,
      "learning_rate": 7.964034505716476e-07,
      "loss": 0.0,
      "reward": 0.25,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.25,
      "step": 192
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.386,
      "grad_norm": 0.009392702020704746,
      "kl": 0.001972198486328125,
      "learning_rate": 7.93768694627233e-07,
      "loss": 0.0001,
      "reward": 0.25,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.25,
      "step": 193
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1788.25,
      "epoch": 0.388,
      "grad_norm": 1.1597681045532227,
      "kl": 0.01373291015625,
      "learning_rate": 7.911220577405484e-07,
      "loss": 0.1207,
      "reward": 0.1875,
      "reward_std": 0.2651650384068489,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.1875,
      "step": 194
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.39,
      "grad_norm": 0.00962373148649931,
      "kl": 0.0009326934814453125,
      "learning_rate": 7.884636689049422e-07,
      "loss": 0.0,
      "reward": 0.25,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.25,
      "step": 195
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2025.0,
      "epoch": 0.392,
      "grad_norm": 0.009538036771118641,
      "kl": 0.00116729736328125,
      "learning_rate": 7.857936576865356e-07,
      "loss": 0.0,
      "reward": 0.25,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.25,
      "step": 196
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.394,
      "grad_norm": 0.01050383411347866,
      "kl": 0.0007953643798828125,
      "learning_rate": 7.831121542179086e-07,
      "loss": 0.0,
      "reward": 0.25,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.25,
      "step": 197
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.396,
      "grad_norm": 0.009053800255060196,
      "kl": 0.001438140869140625,
      "learning_rate": 7.804192891917571e-07,
      "loss": 0.0001,
      "reward": 0.25,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.25,
      "step": 198
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.398,
      "grad_norm": 0.00897100381553173,
      "kl": 0.001312255859375,
      "learning_rate": 7.777151938545235e-07,
      "loss": 0.0001,
      "reward": 0.25,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.25,
      "step": 199
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.4,
      "grad_norm": 0.01025310903787613,
      "kl": 0.001468658447265625,
      "learning_rate": 7.75e-07,
      "loss": 0.0001,
      "reward": 0.25,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.25,
      "step": 200
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.402,
      "grad_norm": 0.054522059857845306,
      "kl": 0.0010223388671875,
      "learning_rate": 7.72273839962904e-07,
      "loss": 0.0,
      "reward": 0.25,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.25,
      "step": 201
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1953.5,
      "epoch": 0.404,
      "grad_norm": 0.037722665816545486,
      "kl": 0.0011157989501953125,
      "learning_rate": 7.695368466124296e-07,
      "loss": 0.0,
      "reward": 0.25,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.25,
      "step": 202
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.406,
      "grad_norm": 0.008431609719991684,
      "kl": 0.0020599365234375,
      "learning_rate": 7.667891533457718e-07,
      "loss": 0.0001,
      "reward": 0.25,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.25,
      "step": 203
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1929.0,
      "epoch": 0.408,
      "grad_norm": 0.9533175826072693,
      "kl": 0.001682281494140625,
      "learning_rate": 7.640308940816239e-07,
      "loss": 0.0001,
      "reward": 0.5625,
      "reward_std": 0.0883883461356163,
      "rewards/accuracy_reward": 0.5,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.0625,
      "step": 204
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2036.0,
      "epoch": 0.41,
      "grad_norm": 0.013962327502667904,
      "kl": 0.0011119842529296875,
      "learning_rate": 7.612622032536507e-07,
      "loss": 0.0,
      "reward": 0.25,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.25,
      "step": 205
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.412,
      "grad_norm": 0.010255957953631878,
      "kl": 0.0009479522705078125,
      "learning_rate": 7.584832158039378e-07,
      "loss": 0.0,
      "reward": 0.25,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.25,
      "step": 206
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.414,
      "grad_norm": 0.01276308298110962,
      "kl": 0.0014495849609375,
      "learning_rate": 7.556940671764124e-07,
      "loss": 0.0001,
      "reward": 0.25,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.25,
      "step": 207
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1809.5,
      "epoch": 0.416,
      "grad_norm": 0.01625184714794159,
      "kl": 0.00093841552734375,
      "learning_rate": 7.528948933102438e-07,
      "loss": 0.0,
      "reward": 0.75,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.5,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.25,
      "step": 208
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.418,
      "grad_norm": 0.010438801720738411,
      "kl": 0.0012836456298828125,
      "learning_rate": 7.500858306332172e-07,
      "loss": 0.0001,
      "reward": 0.25,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.25,
      "step": 209
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.42,
      "grad_norm": 0.011556737124919891,
      "kl": 0.00215911865234375,
      "learning_rate": 7.472670160550848e-07,
      "loss": 0.0001,
      "reward": 0.25,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.25,
      "step": 210
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.422,
      "grad_norm": 0.009882017970085144,
      "kl": 0.0005950927734375,
      "learning_rate": 7.444385869608921e-07,
      "loss": 0.0,
      "reward": 0.25,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.25,
      "step": 211
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1786.0,
      "epoch": 0.424,
      "grad_norm": 0.8873146176338196,
      "kl": 0.002838134765625,
      "learning_rate": 7.416006812042827e-07,
      "loss": -0.0358,
      "reward": 0.375,
      "reward_std": 0.1767766922712326,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.375,
      "step": 212
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.426,
      "grad_norm": 0.01222989521920681,
      "kl": 0.0014190673828125,
      "learning_rate": 7.387534371007797e-07,
      "loss": 0.0001,
      "reward": 0.25,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.25,
      "step": 213
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.428,
      "grad_norm": 0.009303831495344639,
      "kl": 0.00127410888671875,
      "learning_rate": 7.358969934210438e-07,
      "loss": 0.0001,
      "reward": 0.25,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.25,
      "step": 214
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.43,
      "grad_norm": 0.0105022257193923,
      "kl": 0.0008087158203125,
      "learning_rate": 7.330314893841101e-07,
      "loss": 0.0,
      "reward": 0.125,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.125,
      "step": 215
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.432,
      "grad_norm": 0.763167679309845,
      "kl": 0.001544952392578125,
      "learning_rate": 7.301570646506027e-07,
      "loss": 0.0001,
      "reward": 0.3125,
      "reward_std": 0.0883883461356163,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3125,
      "step": 216
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.434,
      "grad_norm": 0.010868730954825878,
      "kl": 0.001415252685546875,
      "learning_rate": 7.27273859315928e-07,
      "loss": 0.0001,
      "reward": 0.25,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.25,
      "step": 217
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.436,
      "grad_norm": 0.016456812620162964,
      "kl": 0.0006694793701171875,
      "learning_rate": 7.243820139034464e-07,
      "loss": 0.0,
      "reward": 0.125,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.125,
      "step": 218
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1947.75,
      "epoch": 0.438,
      "grad_norm": 0.5713726282119751,
      "kl": 0.0039825439453125,
      "learning_rate": 7.214816693576234e-07,
      "loss": 0.0002,
      "reward": 0.1875,
      "reward_std": 0.0883883461356163,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.1875,
      "step": 219
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.44,
      "grad_norm": 0.008825725875794888,
      "kl": 0.000873565673828125,
      "learning_rate": 7.185729670371604e-07,
      "loss": 0.0,
      "reward": 0.25,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.25,
      "step": 220
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.442,
      "grad_norm": 0.015775861218571663,
      "kl": 0.0016326904296875,
      "learning_rate": 7.156560487081051e-07,
      "loss": 0.0001,
      "reward": 0.75,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.5,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.25,
      "step": 221
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1892.75,
      "epoch": 0.444,
      "grad_norm": 0.011794094927608967,
      "kl": 0.002288818359375,
      "learning_rate": 7.127310565369415e-07,
      "loss": 0.0001,
      "reward": 0.25,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.25,
      "step": 222
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.446,
      "grad_norm": 0.8902475833892822,
      "kl": 0.0021953582763671875,
      "learning_rate": 7.097981330836616e-07,
      "loss": 0.0001,
      "reward": 0.1875,
      "reward_std": 0.0883883461356163,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.1875,
      "step": 223
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1976.5,
      "epoch": 0.448,
      "grad_norm": 0.01060924306511879,
      "kl": 0.00159454345703125,
      "learning_rate": 7.068574212948169e-07,
      "loss": 0.0001,
      "reward": 0.75,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.5,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.25,
      "step": 224
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1884.0,
      "epoch": 0.45,
      "grad_norm": 0.7874477505683899,
      "kl": 0.00165557861328125,
      "learning_rate": 7.039090644965509e-07,
      "loss": 0.0675,
      "reward": 0.375,
      "reward_std": 0.1767766922712326,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.375,
      "step": 225
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.452,
      "grad_norm": 0.009480384178459644,
      "kl": 0.001316070556640625,
      "learning_rate": 7.009532063876148e-07,
      "loss": 0.0001,
      "reward": 0.25,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.25,
      "step": 226
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.454,
      "grad_norm": 0.8971606492996216,
      "kl": 0.00217437744140625,
      "learning_rate": 6.979899910323624e-07,
      "loss": 0.0001,
      "reward": 0.1875,
      "reward_std": 0.0883883461356163,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.1875,
      "step": 227
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1950.75,
      "epoch": 0.456,
      "grad_norm": 0.013516876846551895,
      "kl": 0.001598358154296875,
      "learning_rate": 6.950195628537299e-07,
      "loss": 0.0001,
      "reward": 0.25,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.25,
      "step": 228
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.458,
      "grad_norm": 0.012519976124167442,
      "kl": 0.0012359619140625,
      "learning_rate": 6.920420666261961e-07,
      "loss": 0.0,
      "reward": 0.25,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.25,
      "step": 229
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1604.75,
      "epoch": 0.46,
      "grad_norm": 1.0533803701400757,
      "kl": 0.0024871826171875,
      "learning_rate": 6.890576474687263e-07,
      "loss": 0.1645,
      "reward": 0.75,
      "reward_std": 0.7071067541837692,
      "rewards/accuracy_reward": 0.25,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.5,
      "step": 230
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.462,
      "grad_norm": 0.04883728548884392,
      "kl": 0.0012359619140625,
      "learning_rate": 6.860664508377001e-07,
      "loss": 0.0,
      "reward": 0.25,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.25,
      "step": 231
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.464,
      "grad_norm": 0.01617475040256977,
      "kl": 0.00106048583984375,
      "learning_rate": 6.83068622519821e-07,
      "loss": 0.0,
      "reward": 0.25,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.25,
      "step": 232
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.466,
      "grad_norm": 0.01098883431404829,
      "kl": 0.00160980224609375,
      "learning_rate": 6.800643086250121e-07,
      "loss": 0.0001,
      "reward": 0.25,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.25,
      "step": 233
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.468,
      "grad_norm": 0.010283930227160454,
      "kl": 0.001750946044921875,
      "learning_rate": 6.770536555792944e-07,
      "loss": 0.0001,
      "reward": 0.25,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.25,
      "step": 234
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.47,
      "grad_norm": 0.7373048663139343,
      "kl": 0.00131988525390625,
      "learning_rate": 6.740368101176495e-07,
      "loss": 0.0001,
      "reward": 0.1875,
      "reward_std": 0.0883883461356163,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.1875,
      "step": 235
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.472,
      "grad_norm": 0.01158232893794775,
      "kl": 0.001461029052734375,
      "learning_rate": 6.710139192768694e-07,
      "loss": 0.0001,
      "reward": 0.25,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.25,
      "step": 236
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.474,
      "grad_norm": 0.8555125594139099,
      "kl": 0.001068115234375,
      "learning_rate": 6.679851303883891e-07,
      "loss": 0.0,
      "reward": 0.1875,
      "reward_std": 0.0883883461356163,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.1875,
      "step": 237
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.476,
      "grad_norm": 0.8464781641960144,
      "kl": 0.006557464599609375,
      "learning_rate": 6.649505910711058e-07,
      "loss": 0.0003,
      "reward": 0.1875,
      "reward_std": 0.0883883461356163,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.1875,
      "step": 238
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.478,
      "grad_norm": 0.14680787920951843,
      "kl": 0.003086090087890625,
      "learning_rate": 6.619104492241847e-07,
      "loss": 0.0001,
      "reward": 0.25,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.25,
      "step": 239
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.48,
      "grad_norm": 0.8916466236114502,
      "kl": 0.010036468505859375,
      "learning_rate": 6.588648530198504e-07,
      "loss": 0.0004,
      "reward": 0.1875,
      "reward_std": 0.0883883461356163,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.1875,
      "step": 240
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.482,
      "grad_norm": 0.012613446451723576,
      "kl": 0.001827239990234375,
      "learning_rate": 6.558139508961654e-07,
      "loss": 0.0001,
      "reward": 0.25,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.25,
      "step": 241
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.484,
      "grad_norm": 0.012667362578213215,
      "kl": 0.001438140869140625,
      "learning_rate": 6.527578915497951e-07,
      "loss": 0.0001,
      "reward": 0.75,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.5,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.25,
      "step": 242
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.486,
      "grad_norm": 0.011633522808551788,
      "kl": 0.0007953643798828125,
      "learning_rate": 6.496968239287603e-07,
      "loss": 0.0,
      "reward": 0.25,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.25,
      "step": 243
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.488,
      "grad_norm": 0.010322234593331814,
      "kl": 0.00118255615234375,
      "learning_rate": 6.466308972251785e-07,
      "loss": 0.0,
      "reward": 0.25,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.25,
      "step": 244
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.49,
      "grad_norm": 0.013764915056526661,
      "kl": 0.001338958740234375,
      "learning_rate": 6.435602608679916e-07,
      "loss": 0.0001,
      "reward": 0.25,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.25,
      "step": 245
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.492,
      "grad_norm": 0.016423719003796577,
      "kl": 0.0011043548583984375,
      "learning_rate": 6.404850645156841e-07,
      "loss": 0.0,
      "reward": 0.25,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.25,
      "step": 246
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.494,
      "grad_norm": 0.011886836029589176,
      "kl": 0.0008907318115234375,
      "learning_rate": 6.374054580489873e-07,
      "loss": 0.0,
      "reward": 0.25,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.25,
      "step": 247
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.496,
      "grad_norm": 0.013753964565694332,
      "kl": 0.0009822845458984375,
      "learning_rate": 6.343215915635761e-07,
      "loss": 0.0,
      "reward": 0.25,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.25,
      "step": 248
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.498,
      "grad_norm": 0.012257426045835018,
      "kl": 0.00170135498046875,
      "learning_rate": 6.31233615362752e-07,
      "loss": 0.0001,
      "reward": 0.25,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.25,
      "step": 249
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.5,
      "grad_norm": 0.7262430191040039,
      "kl": 0.00194549560546875,
      "learning_rate": 6.281416799501187e-07,
      "loss": 0.0001,
      "reward": 0.1875,
      "reward_std": 0.0883883461356163,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.1875,
      "step": 250
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.502,
      "grad_norm": 0.7270606756210327,
      "kl": 0.0007076263427734375,
      "learning_rate": 6.25045936022246e-07,
      "loss": 0.0,
      "reward": 0.1875,
      "reward_std": 0.0883883461356163,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.1875,
      "step": 251
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.504,
      "grad_norm": 0.017829036340117455,
      "kl": 0.0005779266357421875,
      "learning_rate": 6.219465344613258e-07,
      "loss": 0.0,
      "reward": 0.25,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.25,
      "step": 252
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.506,
      "grad_norm": 0.0317312628030777,
      "kl": 0.0013294219970703125,
      "learning_rate": 6.188436263278172e-07,
      "loss": 0.0001,
      "reward": 0.125,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.125,
      "step": 253
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1972.25,
      "epoch": 0.508,
      "grad_norm": 0.026640823110938072,
      "kl": 0.000835418701171875,
      "learning_rate": 6.157373628530852e-07,
      "loss": 0.0,
      "reward": 0.125,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.125,
      "step": 254
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.51,
      "grad_norm": 0.012969509698450565,
      "kl": 0.001567840576171875,
      "learning_rate": 6.126278954320294e-07,
      "loss": 0.0001,
      "reward": 0.25,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.25,
      "step": 255
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.512,
      "grad_norm": 0.012548292055726051,
      "kl": 0.0010986328125,
      "learning_rate": 6.095153756157051e-07,
      "loss": 0.0,
      "reward": 0.25,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.25,
      "step": 256
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.514,
      "grad_norm": 0.008928977884352207,
      "kl": 0.000873565673828125,
      "learning_rate": 6.06399955103937e-07,
      "loss": 0.0,
      "reward": 0.25,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.25,
      "step": 257
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.516,
      "grad_norm": 0.8101487755775452,
      "kl": 0.0009307861328125,
      "learning_rate": 6.032817857379256e-07,
      "loss": 0.0,
      "reward": 0.1875,
      "reward_std": 0.0883883461356163,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.1875,
      "step": 258
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.518,
      "grad_norm": 0.8978201746940613,
      "kl": 0.0007915496826171875,
      "learning_rate": 6.001610194928464e-07,
      "loss": 0.0,
      "reward": 0.1875,
      "reward_std": 0.0883883461356163,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.1875,
      "step": 259
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.52,
      "grad_norm": 1.092624306678772,
      "kl": 0.00101470947265625,
      "learning_rate": 5.97037808470444e-07,
      "loss": 0.0,
      "reward": 0.125,
      "reward_std": 0.1767766922712326,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.125,
      "step": 260
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.522,
      "grad_norm": 0.7928449511528015,
      "kl": 0.0011920928955078125,
      "learning_rate": 5.939123048916173e-07,
      "loss": 0.0,
      "reward": 0.1875,
      "reward_std": 0.0883883461356163,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.1875,
      "step": 261
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1534.75,
      "epoch": 0.524,
      "grad_norm": 1.0032625198364258,
      "kl": 0.0021257400512695312,
      "learning_rate": 5.907846610890011e-07,
      "loss": 0.0615,
      "reward": 0.375,
      "reward_std": 0.1767766922712326,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.375,
      "step": 262
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.526,
      "grad_norm": 0.011720544658601284,
      "kl": 0.000644683837890625,
      "learning_rate": 5.87655029499542e-07,
      "loss": 0.0,
      "reward": 0.25,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.25,
      "step": 263
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.528,
      "grad_norm": 0.011771513149142265,
      "kl": 0.0007781982421875,
      "learning_rate": 5.845235626570683e-07,
      "loss": 0.0,
      "reward": 0.25,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.25,
      "step": 264
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.53,
      "grad_norm": 0.013503954745829105,
      "kl": 0.001155853271484375,
      "learning_rate": 5.813904131848564e-07,
      "loss": 0.0,
      "reward": 0.25,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.25,
      "step": 265
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.532,
      "grad_norm": 0.09234623610973358,
      "kl": 0.0018482208251953125,
      "learning_rate": 5.78255733788191e-07,
      "loss": 0.0001,
      "reward": 0.25,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.25,
      "step": 266
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.534,
      "grad_norm": 0.011625738814473152,
      "kl": 0.0007114410400390625,
      "learning_rate": 5.751196772469237e-07,
      "loss": 0.0,
      "reward": 0.25,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.25,
      "step": 267
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1589.75,
      "epoch": 0.536,
      "grad_norm": 0.9924006462097168,
      "kl": 0.0024566650390625,
      "learning_rate": 5.71982396408026e-07,
      "loss": -0.0413,
      "reward": 0.875,
      "reward_std": 0.1767766922712326,
      "rewards/accuracy_reward": 0.5,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.375,
      "step": 268
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.538,
      "grad_norm": 0.726823627948761,
      "kl": 0.001861572265625,
      "learning_rate": 5.688440441781398e-07,
      "loss": 0.0001,
      "reward": 0.1875,
      "reward_std": 0.0883883461356163,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.1875,
      "step": 269
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.54,
      "grad_norm": 0.011368845589458942,
      "kl": 0.001186370849609375,
      "learning_rate": 5.657047735161255e-07,
      "loss": 0.0,
      "reward": 0.25,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.25,
      "step": 270
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.542,
      "grad_norm": 0.014150974340736866,
      "kl": 0.0012617111206054688,
      "learning_rate": 5.625647374256061e-07,
      "loss": 0.0001,
      "reward": 0.25,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.25,
      "step": 271
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.544,
      "grad_norm": 0.03309360519051552,
      "kl": 0.00139617919921875,
      "learning_rate": 5.594240889475106e-07,
      "loss": 0.0001,
      "reward": 0.125,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.125,
      "step": 272
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.546,
      "grad_norm": 0.027406711131334305,
      "kl": 0.002048492431640625,
      "learning_rate": 5.562829811526154e-07,
      "loss": 0.0001,
      "reward": 0.75,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.5,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.25,
      "step": 273
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1862.75,
      "epoch": 0.548,
      "grad_norm": 0.872232973575592,
      "kl": 0.001857757568359375,
      "learning_rate": 5.531415671340826e-07,
      "loss": 0.0781,
      "reward": 0.375,
      "reward_std": 0.1767766922712326,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.375,
      "step": 274
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1893.5,
      "epoch": 0.55,
      "grad_norm": 0.013754754327237606,
      "kl": 0.00106048583984375,
      "learning_rate": 5.5e-07,
      "loss": 0.0,
      "reward": 0.75,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.5,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.25,
      "step": 275
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.552,
      "grad_norm": 0.8218861222267151,
      "kl": 0.0015106201171875,
      "learning_rate": 5.468584328659172e-07,
      "loss": 0.0001,
      "reward": 0.1875,
      "reward_std": 0.0883883461356163,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.1875,
      "step": 276
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.554,
      "grad_norm": 0.01286914199590683,
      "kl": 0.001323699951171875,
      "learning_rate": 5.437170188473847e-07,
      "loss": 0.0001,
      "reward": 0.75,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.5,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.25,
      "step": 277
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.556,
      "grad_norm": 0.8357925415039062,
      "kl": 0.00146484375,
      "learning_rate": 5.405759110524894e-07,
      "loss": 0.0001,
      "reward": 0.6875,
      "reward_std": 0.0883883461356163,
      "rewards/accuracy_reward": 0.5,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.1875,
      "step": 278
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.558,
      "grad_norm": 0.011115595698356628,
      "kl": 0.001163482666015625,
      "learning_rate": 5.37435262574394e-07,
      "loss": 0.0,
      "reward": 0.25,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.25,
      "step": 279
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.56,
      "grad_norm": 0.7368820309638977,
      "kl": 0.0016918182373046875,
      "learning_rate": 5.342952264838747e-07,
      "loss": 0.0001,
      "reward": 0.1875,
      "reward_std": 0.0883883461356163,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.1875,
      "step": 280
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1845.5,
      "epoch": 0.562,
      "grad_norm": 0.8080930709838867,
      "kl": 0.002285003662109375,
      "learning_rate": 5.311559558218603e-07,
      "loss": 0.0872,
      "reward": 0.375,
      "reward_std": 0.1767766922712326,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.375,
      "step": 281
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.564,
      "grad_norm": 0.6212303638458252,
      "kl": 0.0017833709716796875,
      "learning_rate": 5.28017603591974e-07,
      "loss": 0.0001,
      "reward": 0.1875,
      "reward_std": 0.0883883461356163,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.1875,
      "step": 282
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.566,
      "grad_norm": 0.8698393106460571,
      "kl": 0.001556396484375,
      "learning_rate": 5.248803227530763e-07,
      "loss": 0.0001,
      "reward": 0.1875,
      "reward_std": 0.0883883461356163,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.1875,
      "step": 283
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.568,
      "grad_norm": 0.6195830702781677,
      "kl": 0.001125335693359375,
      "learning_rate": 5.21744266211809e-07,
      "loss": 0.0,
      "reward": 0.1875,
      "reward_std": 0.0883883461356163,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.1875,
      "step": 284
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.57,
      "grad_norm": 0.014679288491606712,
      "kl": 0.00112152099609375,
      "learning_rate": 5.186095868151436e-07,
      "loss": 0.0,
      "reward": 0.25,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.25,
      "step": 285
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.572,
      "grad_norm": 0.011439023539423943,
      "kl": 0.000751495361328125,
      "learning_rate": 5.154764373429315e-07,
      "loss": 0.0,
      "reward": 0.25,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.25,
      "step": 286
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.574,
      "grad_norm": 0.013943897560238838,
      "kl": 0.00086212158203125,
      "learning_rate": 5.123449705004581e-07,
      "loss": 0.0,
      "reward": 0.125,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.125,
      "step": 287
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.576,
      "grad_norm": 0.011966060847043991,
      "kl": 0.001087188720703125,
      "learning_rate": 5.09215338910999e-07,
      "loss": 0.0,
      "reward": 0.25,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.25,
      "step": 288
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.578,
      "grad_norm": 0.7932072877883911,
      "kl": 0.002170562744140625,
      "learning_rate": 5.060876951083828e-07,
      "loss": 0.0001,
      "reward": 0.1875,
      "reward_std": 0.0883883461356163,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.1875,
      "step": 289
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.58,
      "grad_norm": 0.01169038936495781,
      "kl": 0.001129150390625,
      "learning_rate": 5.02962191529556e-07,
      "loss": 0.0,
      "reward": 0.25,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.25,
      "step": 290
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.582,
      "grad_norm": 0.85643470287323,
      "kl": 0.002471923828125,
      "learning_rate": 4.998389805071536e-07,
      "loss": 0.0001,
      "reward": 0.125,
      "reward_std": 0.1767766922712326,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.125,
      "step": 291
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.584,
      "grad_norm": 1.001603364944458,
      "kl": 0.01709747314453125,
      "learning_rate": 4.967182142620745e-07,
      "loss": 0.0007,
      "reward": 0.1875,
      "reward_std": 0.0883883461356163,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.1875,
      "step": 292
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1886.25,
      "epoch": 0.586,
      "grad_norm": 0.7774127721786499,
      "kl": 0.001667022705078125,
      "learning_rate": 4.93600044896063e-07,
      "loss": 0.0664,
      "reward": 0.375,
      "reward_std": 0.1767766922712326,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.375,
      "step": 293
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.588,
      "grad_norm": 1.0563451051712036,
      "kl": 0.00324249267578125,
      "learning_rate": 4.904846243842949e-07,
      "loss": 0.0001,
      "reward": 0.125,
      "reward_std": 0.1767766922712326,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.125,
      "step": 294
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.59,
      "grad_norm": 0.012082475237548351,
      "kl": 0.00081634521484375,
      "learning_rate": 4.873721045679706e-07,
      "loss": 0.0,
      "reward": 0.25,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.25,
      "step": 295
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2003.5,
      "epoch": 0.592,
      "grad_norm": 0.590258002281189,
      "kl": 0.001506805419921875,
      "learning_rate": 4.842626371469149e-07,
      "loss": 0.0161,
      "reward": 0.625,
      "reward_std": 0.5303300619125366,
      "rewards/accuracy_reward": 0.25,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.375,
      "step": 296
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.594,
      "grad_norm": 0.7182537913322449,
      "kl": 0.00550079345703125,
      "learning_rate": 4.811563736721829e-07,
      "loss": 0.0002,
      "reward": 0.1875,
      "reward_std": 0.0883883461356163,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.1875,
      "step": 297
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1947.25,
      "epoch": 0.596,
      "grad_norm": 1.0634351968765259,
      "kl": 0.0025787353515625,
      "learning_rate": 4.780534655386743e-07,
      "loss": 0.0387,
      "reward": 0.6875,
      "reward_std": 0.6187184080481529,
      "rewards/accuracy_reward": 0.25,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4375,
      "step": 298
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.598,
      "grad_norm": 0.054920367896556854,
      "kl": 0.00203704833984375,
      "learning_rate": 4.749540639777539e-07,
      "loss": 0.0001,
      "reward": 0.125,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.125,
      "step": 299
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.6,
      "grad_norm": 0.012694926001131535,
      "kl": 0.000698089599609375,
      "learning_rate": 4.7185832004988133e-07,
      "loss": 0.0,
      "reward": 0.125,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.125,
      "step": 300
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1795.5,
      "epoch": 0.602,
      "grad_norm": 0.759120523929596,
      "kl": 0.0024261474609375,
      "learning_rate": 4.68766384637248e-07,
      "loss": 0.0001,
      "reward": 0.1875,
      "reward_std": 0.0883883461356163,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.1875,
      "step": 301
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.604,
      "grad_norm": 0.010465751402080059,
      "kl": 0.001453399658203125,
      "learning_rate": 4.656784084364238e-07,
      "loss": 0.0001,
      "reward": 0.25,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.25,
      "step": 302
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1853.0,
      "epoch": 0.606,
      "grad_norm": 0.8468216061592102,
      "kl": 0.00121307373046875,
      "learning_rate": 4.6259454195101267e-07,
      "loss": -0.0831,
      "reward": 0.1875,
      "reward_std": 0.0883883461356163,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.1875,
      "step": 303
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.608,
      "grad_norm": 0.9815247654914856,
      "kl": 0.001689910888671875,
      "learning_rate": 4.59514935484316e-07,
      "loss": 0.0001,
      "reward": 0.125,
      "reward_std": 0.1767766922712326,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.125,
      "step": 304
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1868.75,
      "epoch": 0.61,
      "grad_norm": 0.8037129044532776,
      "kl": 0.000972747802734375,
      "learning_rate": 4.5643973913200837e-07,
      "loss": 0.075,
      "reward": 0.375,
      "reward_std": 0.1767766922712326,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.375,
      "step": 305
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.612,
      "grad_norm": 0.011851584538817406,
      "kl": 0.001220703125,
      "learning_rate": 4.5336910277482155e-07,
      "loss": 0.0,
      "reward": 0.25,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.25,
      "step": 306
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.614,
      "grad_norm": 0.012452667579054832,
      "kl": 0.000675201416015625,
      "learning_rate": 4.503031760712397e-07,
      "loss": 0.0,
      "reward": 0.125,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.125,
      "step": 307
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.616,
      "grad_norm": 0.8061856627464294,
      "kl": 0.001941680908203125,
      "learning_rate": 4.4724210845020494e-07,
      "loss": 0.0001,
      "reward": 0.1875,
      "reward_std": 0.0883883461356163,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.1875,
      "step": 308
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.618,
      "grad_norm": 0.011598404496908188,
      "kl": 0.0009021759033203125,
      "learning_rate": 4.441860491038345e-07,
      "loss": 0.0,
      "reward": 0.25,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.25,
      "step": 309
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.62,
      "grad_norm": 0.013049165718257427,
      "kl": 0.0008697509765625,
      "learning_rate": 4.4113514698014953e-07,
      "loss": 0.0,
      "reward": 0.25,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.25,
      "step": 310
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.622,
      "grad_norm": 0.01000931765884161,
      "kl": 0.0010738372802734375,
      "learning_rate": 4.3808955077581546e-07,
      "loss": 0.0,
      "reward": 0.25,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.25,
      "step": 311
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.624,
      "grad_norm": 0.017103025689721107,
      "kl": 0.00090789794921875,
      "learning_rate": 4.350494089288943e-07,
      "loss": 0.0,
      "reward": 0.25,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.25,
      "step": 312
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.626,
      "grad_norm": 0.015600494109094143,
      "kl": 0.001026153564453125,
      "learning_rate": 4.3201486961161093e-07,
      "loss": 0.0,
      "reward": 0.25,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.25,
      "step": 313
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.628,
      "grad_norm": 0.68843674659729,
      "kl": 0.0009326934814453125,
      "learning_rate": 4.2898608072313045e-07,
      "loss": 0.0,
      "reward": 0.1875,
      "reward_std": 0.0883883461356163,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.1875,
      "step": 314
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.63,
      "grad_norm": 0.02028859592974186,
      "kl": 0.001399993896484375,
      "learning_rate": 4.2596318988235037e-07,
      "loss": 0.0001,
      "reward": 0.25,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.25,
      "step": 315
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.632,
      "grad_norm": 0.010004539042711258,
      "kl": 0.0006055831909179688,
      "learning_rate": 4.2294634442070553e-07,
      "loss": 0.0,
      "reward": 0.25,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.25,
      "step": 316
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.634,
      "grad_norm": 0.012407040223479271,
      "kl": 0.001323699951171875,
      "learning_rate": 4.1993569137498776e-07,
      "loss": 0.0001,
      "reward": 0.25,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.25,
      "step": 317
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.636,
      "grad_norm": 0.7615489363670349,
      "kl": 0.001617431640625,
      "learning_rate": 4.1693137748017915e-07,
      "loss": 0.0001,
      "reward": 0.1875,
      "reward_std": 0.0883883461356163,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.1875,
      "step": 318
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.638,
      "grad_norm": 0.7873152494430542,
      "kl": 0.00064849853515625,
      "learning_rate": 4.1393354916230005e-07,
      "loss": 0.0,
      "reward": 0.1875,
      "reward_std": 0.0883883461356163,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.1875,
      "step": 319
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.64,
      "grad_norm": 0.012965940870344639,
      "kl": 0.000850677490234375,
      "learning_rate": 4.1094235253127374e-07,
      "loss": 0.0,
      "reward": 0.25,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.25,
      "step": 320
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.642,
      "grad_norm": 0.013525367714464664,
      "kl": 0.000804901123046875,
      "learning_rate": 4.079579333738039e-07,
      "loss": 0.0,
      "reward": 0.25,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.25,
      "step": 321
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.644,
      "grad_norm": 0.8479946255683899,
      "kl": 0.0014972686767578125,
      "learning_rate": 4.0498043714627006e-07,
      "loss": 0.0001,
      "reward": 0.1875,
      "reward_std": 0.0883883461356163,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.1875,
      "step": 322
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.646,
      "grad_norm": 0.7696375250816345,
      "kl": 0.002063751220703125,
      "learning_rate": 4.020100089676376e-07,
      "loss": 0.0001,
      "reward": 0.1875,
      "reward_std": 0.0883883461356163,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.1875,
      "step": 323
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.648,
      "grad_norm": 0.012891444377601147,
      "kl": 0.00150299072265625,
      "learning_rate": 3.9904679361238526e-07,
      "loss": 0.0001,
      "reward": 0.25,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.25,
      "step": 324
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.65,
      "grad_norm": 1.1341594457626343,
      "kl": 0.002437591552734375,
      "learning_rate": 3.9609093550344907e-07,
      "loss": 0.0001,
      "reward": 0.375,
      "reward_std": 0.5303300842642784,
      "rewards/accuracy_reward": 0.25,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.125,
      "step": 325
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.652,
      "grad_norm": 0.015124933794140816,
      "kl": 0.0007419586181640625,
      "learning_rate": 3.931425787051832e-07,
      "loss": 0.0,
      "reward": 0.75,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.5,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.25,
      "step": 326
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.654,
      "grad_norm": 0.7730603814125061,
      "kl": 0.002941131591796875,
      "learning_rate": 3.902018669163384e-07,
      "loss": 0.0001,
      "reward": 0.1875,
      "reward_std": 0.0883883461356163,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.1875,
      "step": 327
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.656,
      "grad_norm": 0.02122451364994049,
      "kl": 0.000957489013671875,
      "learning_rate": 3.872689434630585e-07,
      "loss": 0.0,
      "reward": 0.125,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.125,
      "step": 328
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.658,
      "grad_norm": 0.01871700957417488,
      "kl": 0.0010528564453125,
      "learning_rate": 3.843439512918949e-07,
      "loss": 0.0,
      "reward": 0.75,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.5,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.25,
      "step": 329
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.66,
      "grad_norm": 0.0117810582742095,
      "kl": 0.0014801025390625,
      "learning_rate": 3.8142703296283953e-07,
      "loss": 0.0001,
      "reward": 0.25,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.25,
      "step": 330
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1353.5,
      "epoch": 0.662,
      "grad_norm": 0.017292601987719536,
      "kl": 0.0030651092529296875,
      "learning_rate": 3.785183306423767e-07,
      "loss": 0.0001,
      "reward": 0.5,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.5,
      "step": 331
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1955.25,
      "epoch": 0.664,
      "grad_norm": 0.7610009908676147,
      "kl": 0.001689910888671875,
      "learning_rate": 3.7561798609655373e-07,
      "loss": 0.0353,
      "reward": 0.25,
      "reward_std": 0.1767766922712326,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.25,
      "step": 332
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.666,
      "grad_norm": 0.013269172981381416,
      "kl": 0.0008087158203125,
      "learning_rate": 3.72726140684072e-07,
      "loss": 0.0,
      "reward": 0.25,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.25,
      "step": 333
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.668,
      "grad_norm": 0.01594419591128826,
      "kl": 0.000812530517578125,
      "learning_rate": 3.6984293534939737e-07,
      "loss": 0.0,
      "reward": 0.25,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.25,
      "step": 334
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.67,
      "grad_norm": 0.010347824543714523,
      "kl": 0.000751495361328125,
      "learning_rate": 3.6696851061588994e-07,
      "loss": 0.0,
      "reward": 0.25,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.25,
      "step": 335
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.672,
      "grad_norm": 0.014122740365564823,
      "kl": 0.0012054443359375,
      "learning_rate": 3.641030065789562e-07,
      "loss": 0.0,
      "reward": 0.25,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.25,
      "step": 336
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.674,
      "grad_norm": 0.013101037591695786,
      "kl": 0.001033782958984375,
      "learning_rate": 3.612465628992203e-07,
      "loss": 0.0,
      "reward": 0.25,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.25,
      "step": 337
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.676,
      "grad_norm": 0.666260838508606,
      "kl": 0.00324249267578125,
      "learning_rate": 3.5839931879571725e-07,
      "loss": 0.0001,
      "reward": 0.1875,
      "reward_std": 0.0883883461356163,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.1875,
      "step": 338
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2022.75,
      "epoch": 0.678,
      "grad_norm": 0.7250146269798279,
      "kl": 0.00144195556640625,
      "learning_rate": 3.555614130391079e-07,
      "loss": 0.009,
      "reward": 0.375,
      "reward_std": 0.1767766922712326,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.375,
      "step": 339
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2003.25,
      "epoch": 0.68,
      "grad_norm": 0.7383328676223755,
      "kl": 0.00299072265625,
      "learning_rate": 3.5273298394491515e-07,
      "loss": -0.016,
      "reward": 0.1875,
      "reward_std": 0.0883883461356163,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.1875,
      "step": 340
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.682,
      "grad_norm": 0.011795282363891602,
      "kl": 0.000850677490234375,
      "learning_rate": 3.4991416936678276e-07,
      "loss": 0.0,
      "reward": 0.25,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.25,
      "step": 341
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.684,
      "grad_norm": 0.9384401440620422,
      "kl": 0.002079010009765625,
      "learning_rate": 3.471051066897562e-07,
      "loss": 0.0001,
      "reward": 0.1875,
      "reward_std": 0.0883883461356163,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.1875,
      "step": 342
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.686,
      "grad_norm": 0.010377887636423111,
      "kl": 0.0008144378662109375,
      "learning_rate": 3.4430593282358777e-07,
      "loss": 0.0,
      "reward": 0.25,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.25,
      "step": 343
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.688,
      "grad_norm": 0.7026439309120178,
      "kl": 0.001007080078125,
      "learning_rate": 3.4151678419606233e-07,
      "loss": 0.0,
      "reward": 0.375,
      "reward_std": 0.1767766922712326,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.375,
      "step": 344
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.69,
      "grad_norm": 0.011138451285660267,
      "kl": 0.0008373260498046875,
      "learning_rate": 3.387377967463493e-07,
      "loss": 0.0,
      "reward": 0.25,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.25,
      "step": 345
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.692,
      "grad_norm": 0.7246440649032593,
      "kl": 0.0035762786865234375,
      "learning_rate": 3.359691059183761e-07,
      "loss": 0.0001,
      "reward": 0.1875,
      "reward_std": 0.0883883461356163,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.1875,
      "step": 346
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.694,
      "grad_norm": 0.01353926956653595,
      "kl": 0.001087188720703125,
      "learning_rate": 3.3321084665422803e-07,
      "loss": 0.0,
      "reward": 0.25,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.25,
      "step": 347
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1851.25,
      "epoch": 0.696,
      "grad_norm": 1.2388075590133667,
      "kl": 0.00121307373046875,
      "learning_rate": 3.3046315338757026e-07,
      "loss": 0.0841,
      "reward": 0.1875,
      "reward_std": 0.2651650384068489,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.1875,
      "step": 348
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2005.75,
      "epoch": 0.698,
      "grad_norm": 0.7063978314399719,
      "kl": 0.002201080322265625,
      "learning_rate": 3.2772616003709616e-07,
      "loss": 0.0153,
      "reward": 0.375,
      "reward_std": 0.1767766922712326,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.375,
      "step": 349
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.7,
      "grad_norm": 0.009749515913426876,
      "kl": 0.001163482666015625,
      "learning_rate": 3.250000000000001e-07,
      "loss": 0.0,
      "reward": 0.25,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.25,
      "step": 350
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.702,
      "grad_norm": 0.01226428709924221,
      "kl": 0.00107574462890625,
      "learning_rate": 3.222848061454764e-07,
      "loss": 0.0,
      "reward": 0.25,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.25,
      "step": 351
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.704,
      "grad_norm": 0.01303025335073471,
      "kl": 0.001399993896484375,
      "learning_rate": 3.195807108082429e-07,
      "loss": 0.0001,
      "reward": 0.25,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.25,
      "step": 352
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.706,
      "grad_norm": 0.028528396040201187,
      "kl": 0.001117706298828125,
      "learning_rate": 3.168878457820915e-07,
      "loss": 0.0,
      "reward": 0.25,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.25,
      "step": 353
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1785.25,
      "epoch": 0.708,
      "grad_norm": 0.9730527997016907,
      "kl": 0.001430511474609375,
      "learning_rate": 3.142063423134644e-07,
      "loss": 0.122,
      "reward": 0.875,
      "reward_std": 0.1767766922712326,
      "rewards/accuracy_reward": 0.5,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.375,
      "step": 354
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.71,
      "grad_norm": 0.013550005853176117,
      "kl": 0.001338958740234375,
      "learning_rate": 3.115363310950578e-07,
      "loss": 0.0001,
      "reward": 0.25,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.25,
      "step": 355
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.712,
      "grad_norm": 0.010767250321805477,
      "kl": 0.00067138671875,
      "learning_rate": 3.0887794225945143e-07,
      "loss": 0.0,
      "reward": 0.75,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.5,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.25,
      "step": 356
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.714,
      "grad_norm": 0.012552580796182156,
      "kl": 0.00140380859375,
      "learning_rate": 3.062313053727671e-07,
      "loss": 0.0001,
      "reward": 0.25,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.25,
      "step": 357
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.716,
      "grad_norm": 0.6516157984733582,
      "kl": 0.00159454345703125,
      "learning_rate": 3.0359654942835247e-07,
      "loss": 0.0001,
      "reward": 0.1875,
      "reward_std": 0.0883883461356163,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.1875,
      "step": 358
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.718,
      "grad_norm": 0.012717018835246563,
      "kl": 0.0008392333984375,
      "learning_rate": 3.0097380284049523e-07,
      "loss": 0.0,
      "reward": 0.25,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.25,
      "step": 359
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.72,
      "grad_norm": 0.014254845678806305,
      "kl": 0.001018524169921875,
      "learning_rate": 2.9836319343816397e-07,
      "loss": 0.0,
      "reward": 0.25,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.25,
      "step": 360
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.722,
      "grad_norm": 0.017017841339111328,
      "kl": 0.0009784698486328125,
      "learning_rate": 2.9576484845877793e-07,
      "loss": 0.0,
      "reward": 0.25,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.25,
      "step": 361
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1976.75,
      "epoch": 0.724,
      "grad_norm": 0.8839628100395203,
      "kl": 0.002727508544921875,
      "learning_rate": 2.931788945420058e-07,
      "loss": 0.0265,
      "reward": 0.3125,
      "reward_std": 0.2651650309562683,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3125,
      "step": 362
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.726,
      "grad_norm": 0.010562002658843994,
      "kl": 0.00103759765625,
      "learning_rate": 2.9060545772359305e-07,
      "loss": 0.0,
      "reward": 0.25,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.25,
      "step": 363
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.728,
      "grad_norm": 0.013268781825900078,
      "kl": 0.0009174346923828125,
      "learning_rate": 2.8804466342921987e-07,
      "loss": 0.0,
      "reward": 0.25,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.25,
      "step": 364
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1927.75,
      "epoch": 0.73,
      "grad_norm": 0.8109666705131531,
      "kl": 0.0020313262939453125,
      "learning_rate": 2.854966364683872e-07,
      "loss": 0.0471,
      "reward": 0.875,
      "reward_std": 0.1767766922712326,
      "rewards/accuracy_reward": 0.5,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.375,
      "step": 365
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.732,
      "grad_norm": 0.012277526780962944,
      "kl": 0.000827789306640625,
      "learning_rate": 2.829615010283344e-07,
      "loss": 0.0,
      "reward": 0.25,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.25,
      "step": 366
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.734,
      "grad_norm": 0.7054362893104553,
      "kl": 0.0069103240966796875,
      "learning_rate": 2.8043938066798645e-07,
      "loss": 0.0003,
      "reward": 0.6875,
      "reward_std": 0.0883883461356163,
      "rewards/accuracy_reward": 0.5,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.1875,
      "step": 367
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.736,
      "grad_norm": 0.012917861342430115,
      "kl": 0.0012969970703125,
      "learning_rate": 2.7793039831193133e-07,
      "loss": 0.0001,
      "reward": 0.25,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.25,
      "step": 368
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.738,
      "grad_norm": 0.011064048856496811,
      "kl": 0.0007419586181640625,
      "learning_rate": 2.7543467624442956e-07,
      "loss": 0.0,
      "reward": 0.25,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.25,
      "step": 369
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.74,
      "grad_norm": 0.014153026975691319,
      "kl": 0.00079345703125,
      "learning_rate": 2.729523361034538e-07,
      "loss": 0.0,
      "reward": 0.25,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.25,
      "step": 370
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1957.5,
      "epoch": 0.742,
      "grad_norm": 0.02183438278734684,
      "kl": 0.002010345458984375,
      "learning_rate": 2.7048349887476037e-07,
      "loss": 0.0001,
      "reward": 0.25,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.25,
      "step": 371
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1967.0,
      "epoch": 0.744,
      "grad_norm": 0.6841965317726135,
      "kl": 0.00157928466796875,
      "learning_rate": 2.6802828488599294e-07,
      "loss": 0.0304,
      "reward": 0.375,
      "reward_std": 0.1767766922712326,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.375,
      "step": 372
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.746,
      "grad_norm": 0.010570400394499302,
      "kl": 0.00075531005859375,
      "learning_rate": 2.655868138008171e-07,
      "loss": 0.0,
      "reward": 0.25,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.25,
      "step": 373
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.748,
      "grad_norm": 0.013351581990718842,
      "kl": 0.000873565673828125,
      "learning_rate": 2.631592046130896e-07,
      "loss": 0.0,
      "reward": 0.25,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.25,
      "step": 374
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1565.75,
      "epoch": 0.75,
      "grad_norm": 0.9974377751350403,
      "kl": 0.008136749267578125,
      "learning_rate": 2.6074557564105724e-07,
      "loss": -0.3145,
      "reward": 0.1875,
      "reward_std": 0.0883883461356163,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.1875,
      "step": 375
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1881.75,
      "epoch": 0.752,
      "grad_norm": 1.1940348148345947,
      "kl": 0.0038909912109375,
      "learning_rate": 2.583460445215911e-07,
      "loss": 0.0687,
      "reward": 0.3125,
      "reward_std": 0.2651650384068489,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3125,
      "step": 376
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.754,
      "grad_norm": 0.01335156336426735,
      "kl": 0.0006580352783203125,
      "learning_rate": 2.5596072820445254e-07,
      "loss": 0.0,
      "reward": 0.25,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.25,
      "step": 377
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.756,
      "grad_norm": 0.011012405157089233,
      "kl": 0.00142669677734375,
      "learning_rate": 2.5358974294659373e-07,
      "loss": 0.0001,
      "reward": 0.25,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.25,
      "step": 378
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.758,
      "grad_norm": 0.16783788800239563,
      "kl": 0.00443267822265625,
      "learning_rate": 2.512332043064913e-07,
      "loss": 0.0002,
      "reward": 0.125,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.125,
      "step": 379
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.76,
      "grad_norm": 0.011893996968865395,
      "kl": 0.001003265380859375,
      "learning_rate": 2.488912271385139e-07,
      "loss": 0.0,
      "reward": 0.25,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.25,
      "step": 380
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.762,
      "grad_norm": 0.7219942212104797,
      "kl": 0.00760650634765625,
      "learning_rate": 2.465639255873246e-07,
      "loss": 0.0003,
      "reward": 0.1875,
      "reward_std": 0.0883883461356163,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.1875,
      "step": 381
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.764,
      "grad_norm": 0.04678433761000633,
      "kl": 0.0009021759033203125,
      "learning_rate": 2.4425141308231765e-07,
      "loss": 0.0,
      "reward": 0.25,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.25,
      "step": 382
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.766,
      "grad_norm": 0.011990766040980816,
      "kl": 0.0006542205810546875,
      "learning_rate": 2.4195380233209006e-07,
      "loss": 0.0,
      "reward": 0.25,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.25,
      "step": 383
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.768,
      "grad_norm": 0.011609113775193691,
      "kl": 0.0009002685546875,
      "learning_rate": 2.3967120531894857e-07,
      "loss": 0.0,
      "reward": 0.25,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.25,
      "step": 384
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.77,
      "grad_norm": 0.029996510595083237,
      "kl": 0.001007080078125,
      "learning_rate": 2.374037332934512e-07,
      "loss": 0.0,
      "reward": 0.25,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.25,
      "step": 385
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.772,
      "grad_norm": 0.7205003499984741,
      "kl": 0.001007080078125,
      "learning_rate": 2.3515149676898552e-07,
      "loss": 0.0,
      "reward": 0.1875,
      "reward_std": 0.0883883461356163,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.1875,
      "step": 386
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.774,
      "grad_norm": 0.01278019044548273,
      "kl": 0.000644683837890625,
      "learning_rate": 2.3291460551638237e-07,
      "loss": 0.0,
      "reward": 0.25,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.25,
      "step": 387
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.776,
      "grad_norm": 0.050757694989442825,
      "kl": 0.0013141632080078125,
      "learning_rate": 2.306931685585657e-07,
      "loss": 0.0001,
      "reward": 0.75,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.5,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.25,
      "step": 388
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.778,
      "grad_norm": 0.8608619570732117,
      "kl": 0.00222015380859375,
      "learning_rate": 2.2848729416523859e-07,
      "loss": 0.0001,
      "reward": 0.625,
      "reward_std": 0.5303300768136978,
      "rewards/accuracy_reward": 0.25,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.375,
      "step": 389
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.78,
      "grad_norm": 0.01890842616558075,
      "kl": 0.0016078948974609375,
      "learning_rate": 2.2629708984760706e-07,
      "loss": 0.0001,
      "reward": 0.25,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.25,
      "step": 390
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1757.5,
      "epoch": 0.782,
      "grad_norm": 0.9615358710289001,
      "kl": 0.00257110595703125,
      "learning_rate": 2.2412266235313973e-07,
      "loss": 0.1401,
      "reward": 0.375,
      "reward_std": 0.1767766922712326,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.375,
      "step": 391
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2038.25,
      "epoch": 0.784,
      "grad_norm": 0.7946398854255676,
      "kl": 0.0009212493896484375,
      "learning_rate": 2.2196411766036487e-07,
      "loss": 0.0034,
      "reward": 0.8125,
      "reward_std": 0.0883883461356163,
      "rewards/accuracy_reward": 0.5,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3125,
      "step": 392
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.786,
      "grad_norm": 0.01277601532638073,
      "kl": 0.001102447509765625,
      "learning_rate": 2.1982156097370557e-07,
      "loss": 0.0,
      "reward": 0.25,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.25,
      "step": 393
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1800.5,
      "epoch": 0.788,
      "grad_norm": 0.9031627774238586,
      "kl": 0.00704193115234375,
      "learning_rate": 2.1769509671835223e-07,
      "loss": 0.1129,
      "reward": 0.375,
      "reward_std": 0.1767766922712326,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.375,
      "step": 394
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.79,
      "grad_norm": 0.017206581309437752,
      "kl": 0.001468658447265625,
      "learning_rate": 2.1558482853517253e-07,
      "loss": 0.0001,
      "reward": 0.25,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.25,
      "step": 395
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1820.75,
      "epoch": 0.792,
      "grad_norm": 0.9008931517601013,
      "kl": 0.0047149658203125,
      "learning_rate": 2.134908592756607e-07,
      "loss": 0.0002,
      "reward": 0.0625,
      "reward_std": 0.0883883461356163,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.0625,
      "step": 396
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2010.25,
      "epoch": 0.794,
      "grad_norm": 0.8031813502311707,
      "kl": 0.00140380859375,
      "learning_rate": 2.1141329099692406e-07,
      "loss": -0.0135,
      "reward": 0.1875,
      "reward_std": 0.0883883461356163,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.1875,
      "step": 397
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.796,
      "grad_norm": 0.015710551291704178,
      "kl": 0.0016117095947265625,
      "learning_rate": 2.0935222495670968e-07,
      "loss": 0.0001,
      "reward": 0.25,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.25,
      "step": 398
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.798,
      "grad_norm": 0.7256066203117371,
      "kl": 0.00640869140625,
      "learning_rate": 2.0730776160846853e-07,
      "loss": 0.0003,
      "reward": 0.1875,
      "reward_std": 0.0883883461356163,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.1875,
      "step": 399
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.8,
      "grad_norm": 0.011895825155079365,
      "kl": 0.00131988525390625,
      "learning_rate": 2.0528000059645995e-07,
      "loss": 0.0001,
      "reward": 0.25,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.25,
      "step": 400
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.802,
      "grad_norm": 0.01262570358812809,
      "kl": 0.00177001953125,
      "learning_rate": 2.032690407508949e-07,
      "loss": 0.0001,
      "reward": 0.25,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.25,
      "step": 401
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.804,
      "grad_norm": 0.014729364775121212,
      "kl": 0.00090789794921875,
      "learning_rate": 2.0127498008311922e-07,
      "loss": 0.0,
      "reward": 0.25,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.25,
      "step": 402
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.806,
      "grad_norm": 0.012951449491083622,
      "kl": 0.00146484375,
      "learning_rate": 1.9929791578083655e-07,
      "loss": 0.0001,
      "reward": 0.25,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.25,
      "step": 403
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.808,
      "grad_norm": 0.6974371671676636,
      "kl": 0.0015716552734375,
      "learning_rate": 1.9733794420337213e-07,
      "loss": 0.0001,
      "reward": 0.375,
      "reward_std": 0.1767766922712326,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.375,
      "step": 404
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.81,
      "grad_norm": 0.012584330514073372,
      "kl": 0.001598358154296875,
      "learning_rate": 1.9539516087697517e-07,
      "loss": 0.0001,
      "reward": 0.25,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.25,
      "step": 405
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.812,
      "grad_norm": 0.011754573322832584,
      "kl": 0.00107574462890625,
      "learning_rate": 1.934696604901642e-07,
      "loss": 0.0,
      "reward": 0.25,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.25,
      "step": 406
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.814,
      "grad_norm": 0.02457410842180252,
      "kl": 0.0015869140625,
      "learning_rate": 1.915615368891117e-07,
      "loss": 0.0001,
      "reward": 0.25,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.25,
      "step": 407
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.816,
      "grad_norm": 0.7841270565986633,
      "kl": 0.00121307373046875,
      "learning_rate": 1.8967088307307e-07,
      "loss": 0.0,
      "reward": 0.375,
      "reward_std": 0.1767766922712326,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.375,
      "step": 408
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.818,
      "grad_norm": 0.01260992232710123,
      "kl": 0.0014495849609375,
      "learning_rate": 1.8779779118983867e-07,
      "loss": 0.0001,
      "reward": 0.25,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.25,
      "step": 409
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.82,
      "grad_norm": 0.011464129202067852,
      "kl": 0.0010471343994140625,
      "learning_rate": 1.8594235253127372e-07,
      "loss": 0.0,
      "reward": 0.25,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.25,
      "step": 410
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1915.0,
      "epoch": 0.822,
      "grad_norm": 0.8047151565551758,
      "kl": 0.00250244140625,
      "learning_rate": 1.8410465752883758e-07,
      "loss": -0.0527,
      "reward": 0.1875,
      "reward_std": 0.0883883461356163,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.1875,
      "step": 411
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.824,
      "grad_norm": 0.0124558350071311,
      "kl": 0.00130462646484375,
      "learning_rate": 1.822847957491922e-07,
      "loss": 0.0001,
      "reward": 0.25,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.25,
      "step": 412
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.826,
      "grad_norm": 0.01380992028862238,
      "kl": 0.0008144378662109375,
      "learning_rate": 1.804828558898332e-07,
      "loss": 0.0,
      "reward": 0.125,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.125,
      "step": 413
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.828,
      "grad_norm": 0.9551144242286682,
      "kl": 0.011430740356445312,
      "learning_rate": 1.7869892577476722e-07,
      "loss": 0.0005,
      "reward": 0.25,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.25,
      "step": 414
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.83,
      "grad_norm": 0.012345471419394016,
      "kl": 0.001438140869140625,
      "learning_rate": 1.7693309235023127e-07,
      "loss": 0.0001,
      "reward": 0.25,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.25,
      "step": 415
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1821.0,
      "epoch": 0.832,
      "grad_norm": 0.7977136969566345,
      "kl": 0.001773834228515625,
      "learning_rate": 1.7518544168045524e-07,
      "loss": 0.0981,
      "reward": 0.375,
      "reward_std": 0.1767766922712326,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.375,
      "step": 416
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.834,
      "grad_norm": 0.011019429191946983,
      "kl": 0.0006771087646484375,
      "learning_rate": 1.7345605894346726e-07,
      "loss": 0.0,
      "reward": 0.25,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.25,
      "step": 417
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.836,
      "grad_norm": 0.015206689946353436,
      "kl": 0.0016021728515625,
      "learning_rate": 1.7174502842694212e-07,
      "loss": 0.0001,
      "reward": 0.25,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.25,
      "step": 418
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.838,
      "grad_norm": 0.04012497141957283,
      "kl": 0.000911712646484375,
      "learning_rate": 1.7005243352409333e-07,
      "loss": 0.0,
      "reward": 0.25,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.25,
      "step": 419
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.84,
      "grad_norm": 0.01337823923677206,
      "kl": 0.0009021759033203125,
      "learning_rate": 1.6837835672960831e-07,
      "loss": 0.0,
      "reward": 0.25,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.25,
      "step": 420
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.842,
      "grad_norm": 0.010002349503338337,
      "kl": 0.0014495849609375,
      "learning_rate": 1.6672287963562852e-07,
      "loss": 0.0001,
      "reward": 0.25,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.25,
      "step": 421
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.844,
      "grad_norm": 0.716344952583313,
      "kl": 0.00653076171875,
      "learning_rate": 1.6508608292777203e-07,
      "loss": 0.0003,
      "reward": 0.1875,
      "reward_std": 0.0883883461356163,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.1875,
      "step": 422
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.846,
      "grad_norm": 0.037377193570137024,
      "kl": 0.001346588134765625,
      "learning_rate": 1.6346804638120098e-07,
      "loss": 0.0001,
      "reward": 0.25,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.25,
      "step": 423
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.848,
      "grad_norm": 0.011378524824976921,
      "kl": 0.0007991790771484375,
      "learning_rate": 1.6186884885673413e-07,
      "loss": 0.0,
      "reward": 0.25,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.25,
      "step": 424
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.85,
      "grad_norm": 0.012089181691408157,
      "kl": 0.00154876708984375,
      "learning_rate": 1.6028856829700258e-07,
      "loss": 0.0001,
      "reward": 0.25,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.25,
      "step": 425
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.852,
      "grad_norm": 0.7250344753265381,
      "kl": 0.002223968505859375,
      "learning_rate": 1.5872728172265146e-07,
      "loss": 0.0001,
      "reward": 0.1875,
      "reward_std": 0.0883883461356163,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.1875,
      "step": 426
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1880.5,
      "epoch": 0.854,
      "grad_norm": 0.8333255648612976,
      "kl": 0.0015106201171875,
      "learning_rate": 1.5718506522858572e-07,
      "loss": 0.0692,
      "reward": 0.375,
      "reward_std": 0.1767766922712326,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.375,
      "step": 427
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1326.75,
      "epoch": 0.856,
      "grad_norm": 1.4510780572891235,
      "kl": 0.0024261474609375,
      "learning_rate": 1.5566199398026147e-07,
      "loss": 0.0319,
      "reward": 0.375,
      "reward_std": 0.1767766922712326,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.375,
      "step": 428
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.858,
      "grad_norm": 0.01227201521396637,
      "kl": 0.0006866455078125,
      "learning_rate": 1.5415814221002265e-07,
      "loss": 0.0,
      "reward": 0.25,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.25,
      "step": 429
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.86,
      "grad_norm": 0.012068121694028378,
      "kl": 0.0009098052978515625,
      "learning_rate": 1.5267358321348285e-07,
      "loss": 0.0,
      "reward": 0.25,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.25,
      "step": 430
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.862,
      "grad_norm": 0.012486455962061882,
      "kl": 0.000980377197265625,
      "learning_rate": 1.5120838934595337e-07,
      "loss": 0.0,
      "reward": 0.25,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.25,
      "step": 431
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.864,
      "grad_norm": 0.013689450919628143,
      "kl": 0.00124359130859375,
      "learning_rate": 1.4976263201891613e-07,
      "loss": 0.0,
      "reward": 0.25,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.25,
      "step": 432
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.866,
      "grad_norm": 0.5863283276557922,
      "kl": 0.00223541259765625,
      "learning_rate": 1.483363816965435e-07,
      "loss": 0.0001,
      "reward": 0.1875,
      "reward_std": 0.0883883461356163,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.1875,
      "step": 433
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.868,
      "grad_norm": 0.01356459315866232,
      "kl": 0.00078582763671875,
      "learning_rate": 1.469297078922642e-07,
      "loss": 0.0,
      "reward": 0.25,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.25,
      "step": 434
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.87,
      "grad_norm": 0.009759028442203999,
      "kl": 0.0011749267578125,
      "learning_rate": 1.4554267916537495e-07,
      "loss": 0.0,
      "reward": 0.25,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.25,
      "step": 435
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.872,
      "grad_norm": 0.02292313612997532,
      "kl": 0.0009002685546875,
      "learning_rate": 1.4417536311769885e-07,
      "loss": 0.0,
      "reward": 0.25,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.25,
      "step": 436
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.874,
      "grad_norm": 0.013289421796798706,
      "kl": 0.00077056884765625,
      "learning_rate": 1.4282782639029128e-07,
      "loss": 0.0,
      "reward": 0.25,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.25,
      "step": 437
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.876,
      "grad_norm": 0.0265584085136652,
      "kl": 0.00119781494140625,
      "learning_rate": 1.4150013466019114e-07,
      "loss": 0.0,
      "reward": 0.25,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.25,
      "step": 438
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2008.25,
      "epoch": 0.878,
      "grad_norm": 0.013609832152724266,
      "kl": 0.000782012939453125,
      "learning_rate": 1.4019235263722034e-07,
      "loss": 0.0,
      "reward": 0.25,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.25,
      "step": 439
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.88,
      "grad_norm": 0.010987906716763973,
      "kl": 0.00115203857421875,
      "learning_rate": 1.3890454406082956e-07,
      "loss": 0.0,
      "reward": 0.25,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.25,
      "step": 440
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.882,
      "grad_norm": 0.01149643212556839,
      "kl": 0.0007152557373046875,
      "learning_rate": 1.3763677169699217e-07,
      "loss": 0.0,
      "reward": 0.75,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.5,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.25,
      "step": 441
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.884,
      "grad_norm": 0.012063896283507347,
      "kl": 0.000858306884765625,
      "learning_rate": 1.3638909733514452e-07,
      "loss": 0.0,
      "reward": 0.25,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.25,
      "step": 442
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.886,
      "grad_norm": 0.6539027094841003,
      "kl": 0.0033817291259765625,
      "learning_rate": 1.351615817851748e-07,
      "loss": 0.0001,
      "reward": 0.1875,
      "reward_std": 0.0883883461356163,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.1875,
      "step": 443
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.888,
      "grad_norm": 0.683651864528656,
      "kl": 0.001430511474609375,
      "learning_rate": 1.3395428487445914e-07,
      "loss": 0.0001,
      "reward": 0.1875,
      "reward_std": 0.0883883461356163,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.1875,
      "step": 444
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.89,
      "grad_norm": 0.014244834892451763,
      "kl": 0.001583099365234375,
      "learning_rate": 1.3276726544494571e-07,
      "loss": 0.0001,
      "reward": 0.25,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.25,
      "step": 445
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1551.25,
      "epoch": 0.892,
      "grad_norm": 0.04096902534365654,
      "kl": 0.003017425537109375,
      "learning_rate": 1.316005813502869e-07,
      "loss": 0.0001,
      "reward": 0.5,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.5,
      "step": 446
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.894,
      "grad_norm": 0.013455760665237904,
      "kl": 0.00115203857421875,
      "learning_rate": 1.3045428945301953e-07,
      "loss": 0.0,
      "reward": 0.25,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.25,
      "step": 447
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1822.0,
      "epoch": 0.896,
      "grad_norm": 1.1271547079086304,
      "kl": 0.0069122314453125,
      "learning_rate": 1.2932844562179352e-07,
      "loss": -0.0998,
      "reward": 0.125,
      "reward_std": 0.1767766922712326,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.125,
      "step": 448
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.898,
      "grad_norm": 0.6208535432815552,
      "kl": 0.0009765625,
      "learning_rate": 1.2822310472864885e-07,
      "loss": 0.0,
      "reward": 0.5,
      "reward_std": 0.3535533845424652,
      "rewards/accuracy_reward": 0.25,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.25,
      "step": 449
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.9,
      "grad_norm": 0.009892701171338558,
      "kl": 0.0006608963012695312,
      "learning_rate": 1.2713832064634125e-07,
      "loss": 0.0,
      "reward": 0.25,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.25,
      "step": 450
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.902,
      "grad_norm": 0.0119470888748765,
      "kl": 0.000751495361328125,
      "learning_rate": 1.260741462457165e-07,
      "loss": 0.0,
      "reward": 0.25,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.25,
      "step": 451
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1717.0,
      "epoch": 0.904,
      "grad_norm": 0.015732314437627792,
      "kl": 0.001445770263671875,
      "learning_rate": 1.2503063339313356e-07,
      "loss": 0.0,
      "reward": 0.125,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.125,
      "step": 452
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.906,
      "grad_norm": 0.7161591649055481,
      "kl": 0.001621246337890625,
      "learning_rate": 1.2400783294793668e-07,
      "loss": 0.0001,
      "reward": 0.1875,
      "reward_std": 0.0883883461356163,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.1875,
      "step": 453
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.908,
      "grad_norm": 0.009790318086743355,
      "kl": 0.0006418228149414062,
      "learning_rate": 1.2300579475997657e-07,
      "loss": 0.0,
      "reward": 0.25,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.25,
      "step": 454
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.91,
      "grad_norm": 0.01178540289402008,
      "kl": 0.0012798309326171875,
      "learning_rate": 1.220245676671809e-07,
      "loss": 0.0001,
      "reward": 0.25,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.25,
      "step": 455
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1967.75,
      "epoch": 0.912,
      "grad_norm": 0.918179452419281,
      "kl": 0.00170135498046875,
      "learning_rate": 1.2106419949317388e-07,
      "loss": 0.0301,
      "reward": 0.375,
      "reward_std": 0.1767766922712326,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.375,
      "step": 456
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.914,
      "grad_norm": 0.7712357640266418,
      "kl": 0.00215911865234375,
      "learning_rate": 1.2012473704494537e-07,
      "loss": 0.0001,
      "reward": 0.6875,
      "reward_std": 0.0883883461356163,
      "rewards/accuracy_reward": 0.5,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.1875,
      "step": 457
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1705.5,
      "epoch": 0.916,
      "grad_norm": 0.8978808522224426,
      "kl": 0.00180816650390625,
      "learning_rate": 1.1920622611056974e-07,
      "loss": 0.0779,
      "reward": 0.25,
      "reward_std": 0.1767766922712326,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.25,
      "step": 458
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.918,
      "grad_norm": 0.8227681517601013,
      "kl": 0.00136566162109375,
      "learning_rate": 1.1830871145697412e-07,
      "loss": 0.0001,
      "reward": 0.1875,
      "reward_std": 0.0883883461356163,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.1875,
      "step": 459
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.92,
      "grad_norm": 0.009116187691688538,
      "kl": 0.00128936767578125,
      "learning_rate": 1.1743223682775649e-07,
      "loss": 0.0001,
      "reward": 0.25,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.25,
      "step": 460
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.922,
      "grad_norm": 0.014834447763860226,
      "kl": 0.00095367431640625,
      "learning_rate": 1.1657684494105386e-07,
      "loss": 0.0,
      "reward": 0.25,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.25,
      "step": 461
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.924,
      "grad_norm": 0.013181531801819801,
      "kl": 0.0008907318115234375,
      "learning_rate": 1.1574257748745986e-07,
      "loss": 0.0,
      "reward": 0.25,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.25,
      "step": 462
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.926,
      "grad_norm": 0.010957694612443447,
      "kl": 0.0006504058837890625,
      "learning_rate": 1.1492947512799328e-07,
      "loss": 0.0,
      "reward": 0.25,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.25,
      "step": 463
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.928,
      "grad_norm": 0.8815539479255676,
      "kl": 0.0010204315185546875,
      "learning_rate": 1.1413757749211602e-07,
      "loss": 0.0,
      "reward": 0.1875,
      "reward_std": 0.0883883461356163,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.1875,
      "step": 464
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.93,
      "grad_norm": 0.010624675080180168,
      "kl": 0.0008087158203125,
      "learning_rate": 1.1336692317580158e-07,
      "loss": 0.0,
      "reward": 0.25,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.25,
      "step": 465
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.932,
      "grad_norm": 0.013844382017850876,
      "kl": 0.001033782958984375,
      "learning_rate": 1.1261754973965422e-07,
      "loss": 0.0,
      "reward": 0.25,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.25,
      "step": 466
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.934,
      "grad_norm": 0.012338408268988132,
      "kl": 0.000690460205078125,
      "learning_rate": 1.1188949370707787e-07,
      "loss": 0.0,
      "reward": 0.25,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.25,
      "step": 467
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1994.0,
      "epoch": 0.936,
      "grad_norm": 0.7254545092582703,
      "kl": 0.0010471343994140625,
      "learning_rate": 1.1118279056249653e-07,
      "loss": 0.0197,
      "reward": 0.375,
      "reward_std": 0.1767766922712326,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.375,
      "step": 468
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.938,
      "grad_norm": 0.01179241482168436,
      "kl": 0.0006885528564453125,
      "learning_rate": 1.1049747474962444e-07,
      "loss": 0.0,
      "reward": 0.25,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.25,
      "step": 469
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.94,
      "grad_norm": 0.6890900135040283,
      "kl": 0.001373291015625,
      "learning_rate": 1.0983357966978745e-07,
      "loss": 0.0001,
      "reward": 0.1875,
      "reward_std": 0.0883883461356163,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.1875,
      "step": 470
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.942,
      "grad_norm": 0.5639128684997559,
      "kl": 0.00131988525390625,
      "learning_rate": 1.0919113768029517e-07,
      "loss": 0.0001,
      "reward": 0.1875,
      "reward_std": 0.0883883461356163,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.1875,
      "step": 471
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.944,
      "grad_norm": 0.012823720462620258,
      "kl": 0.001026153564453125,
      "learning_rate": 1.0857018009286381e-07,
      "loss": 0.0,
      "reward": 0.25,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.25,
      "step": 472
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.946,
      "grad_norm": 0.009964537806808949,
      "kl": 0.0007228851318359375,
      "learning_rate": 1.0797073717209013e-07,
      "loss": 0.0,
      "reward": 0.25,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.25,
      "step": 473
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.948,
      "grad_norm": 0.7185282707214355,
      "kl": 0.0028629302978515625,
      "learning_rate": 1.0739283813397639e-07,
      "loss": 0.0001,
      "reward": 0.1875,
      "reward_std": 0.0883883461356163,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.1875,
      "step": 474
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.95,
      "grad_norm": 0.012274986132979393,
      "kl": 0.00087738037109375,
      "learning_rate": 1.068365111445064e-07,
      "loss": 0.0,
      "reward": 0.25,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.25,
      "step": 475
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1744.25,
      "epoch": 0.952,
      "grad_norm": 0.8758648633956909,
      "kl": 0.002716064453125,
      "learning_rate": 1.063017833182728e-07,
      "loss": 0.1492,
      "reward": 0.1875,
      "reward_std": 0.2651650309562683,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.1875,
      "step": 476
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.954,
      "grad_norm": 0.014645918272435665,
      "kl": 0.000720977783203125,
      "learning_rate": 1.0578868071715544e-07,
      "loss": 0.0,
      "reward": 0.25,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.25,
      "step": 477
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.956,
      "grad_norm": 0.013319121673703194,
      "kl": 0.0009250640869140625,
      "learning_rate": 1.0529722834905125e-07,
      "loss": 0.0,
      "reward": 0.25,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.25,
      "step": 478
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.958,
      "grad_norm": 0.009549077600240707,
      "kl": 0.0014190673828125,
      "learning_rate": 1.0482745016665526e-07,
      "loss": 0.0001,
      "reward": 0.25,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.25,
      "step": 479
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.96,
      "grad_norm": 0.015292295254766941,
      "kl": 0.0009307861328125,
      "learning_rate": 1.0437936906629334e-07,
      "loss": 0.0,
      "reward": 0.25,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.25,
      "step": 480
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.962,
      "grad_norm": 1.031322717666626,
      "kl": 0.003509521484375,
      "learning_rate": 1.0395300688680625e-07,
      "loss": 0.0001,
      "reward": 0.125,
      "reward_std": 0.1767766922712326,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.125,
      "step": 481
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.964,
      "grad_norm": 0.011882193386554718,
      "kl": 0.001026153564453125,
      "learning_rate": 1.0354838440848501e-07,
      "loss": 0.0,
      "reward": 0.25,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.25,
      "step": 482
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1847.0,
      "epoch": 0.966,
      "grad_norm": 0.6594410538673401,
      "kl": 0.002811431884765625,
      "learning_rate": 1.0316552135205837e-07,
      "loss": 0.0864,
      "reward": 0.4375,
      "reward_std": 0.0883883461356163,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4375,
      "step": 483
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.968,
      "grad_norm": 0.013277575373649597,
      "kl": 0.000789642333984375,
      "learning_rate": 1.0280443637773163e-07,
      "loss": 0.0,
      "reward": 0.25,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.25,
      "step": 484
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.97,
      "grad_norm": 0.7425467371940613,
      "kl": 0.00323486328125,
      "learning_rate": 1.0246514708427701e-07,
      "loss": 0.0001,
      "reward": 0.0625,
      "reward_std": 0.0883883461356163,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.0625,
      "step": 485
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.972,
      "grad_norm": 0.01078079268336296,
      "kl": 0.00154876708984375,
      "learning_rate": 1.0214767000817596e-07,
      "loss": 0.0001,
      "reward": 0.25,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.25,
      "step": 486
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.974,
      "grad_norm": 0.012354187667369843,
      "kl": 0.0005855560302734375,
      "learning_rate": 1.0185202062281336e-07,
      "loss": 0.0,
      "reward": 0.25,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.25,
      "step": 487
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.976,
      "grad_norm": 0.7250139117240906,
      "kl": 0.0037212371826171875,
      "learning_rate": 1.0157821333772304e-07,
      "loss": 0.0001,
      "reward": 0.1875,
      "reward_std": 0.0883883461356163,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.1875,
      "step": 488
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.978,
      "grad_norm": 0.014553495682775974,
      "kl": 0.00141143798828125,
      "learning_rate": 1.013262614978859e-07,
      "loss": 0.0001,
      "reward": 0.75,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.5,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.25,
      "step": 489
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.98,
      "grad_norm": 0.01307358592748642,
      "kl": 0.000835418701171875,
      "learning_rate": 1.0109617738307911e-07,
      "loss": 0.0,
      "reward": 0.25,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.25,
      "step": 490
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1908.5,
      "epoch": 0.982,
      "grad_norm": 0.7941016554832458,
      "kl": 0.0010204315185546875,
      "learning_rate": 1.0088797220727779e-07,
      "loss": 0.0558,
      "reward": 0.375,
      "reward_std": 0.1767766922712326,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.375,
      "step": 491
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.984,
      "grad_norm": 0.008943392895162106,
      "kl": 0.000537872314453125,
      "learning_rate": 1.0070165611810855e-07,
      "loss": 0.0,
      "reward": 0.25,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.25,
      "step": 492
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.986,
      "grad_norm": 0.012137016281485558,
      "kl": 0.001377105712890625,
      "learning_rate": 1.005372381963547e-07,
      "loss": 0.0001,
      "reward": 0.25,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.25,
      "step": 493
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.988,
      "grad_norm": 0.01392819918692112,
      "kl": 0.0009784698486328125,
      "learning_rate": 1.0039472645551372e-07,
      "loss": 0.0,
      "reward": 0.25,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.25,
      "step": 494
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.99,
      "grad_norm": 0.010369054041802883,
      "kl": 0.000942230224609375,
      "learning_rate": 1.002741278414069e-07,
      "loss": 0.0,
      "reward": 0.25,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.25,
      "step": 495
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.992,
      "grad_norm": 0.011974328197538853,
      "kl": 0.0006275177001953125,
      "learning_rate": 1.0017544823184055e-07,
      "loss": 0.0,
      "reward": 0.25,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.25,
      "step": 496
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.994,
      "grad_norm": 0.01267942413687706,
      "kl": 0.00104522705078125,
      "learning_rate": 1.0009869243631952e-07,
      "loss": 0.0,
      "reward": 0.25,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.25,
      "step": 497
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.996,
      "grad_norm": 0.876571536064148,
      "kl": 0.003925323486328125,
      "learning_rate": 1.000438641958131e-07,
      "loss": 0.0002,
      "reward": 0.1875,
      "reward_std": 0.0883883461356163,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.1875,
      "step": 498
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.998,
      "grad_norm": 0.6709615588188171,
      "kl": 0.001743316650390625,
      "learning_rate": 1.0001096618257236e-07,
      "loss": 0.0001,
      "reward": 0.1875,
      "reward_std": 0.0883883461356163,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.1875,
      "step": 499
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 1.0,
      "grad_norm": 0.014002146199345589,
      "kl": 0.0007781982421875,
      "learning_rate": 1e-07,
      "loss": 0.0,
      "reward": 0.25,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.25,
      "step": 500
    },
    {
      "epoch": 1.0,
      "step": 500,
      "total_flos": 0.0,
      "train_loss": 0.0033100851627775683,
      "train_runtime": 14646.384,
      "train_samples_per_second": 0.068,
      "train_steps_per_second": 0.034
    }
  ],
  "logging_steps": 1,
  "max_steps": 500,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 1,
  "save_steps": 500,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": true
      },
      "attributes": {}
    }
  },
  "total_flos": 0.0,
  "train_batch_size": 2,
  "trial_name": null,
  "trial_params": null
}