diff --git "a/wandb/run-20241226_145403-c7ovtvny/files/output.log" "b/wandb/run-20241226_145403-c7ovtvny/files/output.log" new file mode 100644--- /dev/null +++ "b/wandb/run-20241226_145403-c7ovtvny/files/output.log" @@ -0,0 +1,866 @@ +/scratch3/workspace/ctpham_umass_edu-ft/envs/prolong-final/lib/python3.10/site-packages/transformers/trainer.py:2833: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. + checkpoint_rng_state = torch.load(rng_file) +/scratch3/workspace/ctpham_umass_edu-ft/envs/prolong-final/lib/python3.10/site-packages/torch/utils/checkpoint.py:1399: FutureWarning: `torch.cpu.amp.autocast(args...)` is deprecated. Please use `torch.amp.autocast('cpu', args...)` instead. + with device_autocast_ctx, torch.cpu.amp.autocast(**cpu_autocast_kwargs), recompute_context: # type: ignore[attr-defined] +[INFO|trainer.py:175] 2024-12-26 14:54:56,722 >> {'loss': 0.4364, 'grad_norm': 17.1519832611084, 'learning_rate': 3.6768177505930655e-07, 'epoch': 0.00027122321670735016, 'num_input_tokens_seen': 5035261952, 'completed': '65.12% (2_401 / 3_687)', 'remaining time': '16:03:40', 'throughput': '2915.21', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 14:55:28,853 >> {'loss': 0.5551, 'grad_norm': 19.901262283325195, 'learning_rate': 3.67312776568654e-07, 'epoch': 0.0005424464334147003, 'num_input_tokens_seen': 5037359104, 'completed': '65.15% (2_402 / 3_687)', 'remaining time': '13:45:31', 'throughput': '8158.65', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 14:55:57,560 >> {'loss': 0.624, 'grad_norm': 20.185373306274414, 'learning_rate': 3.669439250977272e-07, 'epoch': 0.0008136696501220504, 'num_input_tokens_seen': 5039456256, 'completed': '65.17% (2_403 / 3_687)', 'remaining time': '12:34:42', 'throughput': '9131.76', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 14:56:26,588 >> {'loss': 0.5393, 'grad_norm': 17.927940368652344, 'learning_rate': 3.665752209433637e-07, 'epoch': 0.0010848928668294006, 'num_input_tokens_seen': 5041553408, 'completed': '65.20% (2_404 / 3_687)', 'remaining time': '12:00:45', 'throughput': '9030.55', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 14:56:56,750 >> {'loss': 0.3492, 'grad_norm': 13.626057624816895, 'learning_rate': 3.6620666440228254e-07, 'epoch': 0.0013561160835367507, 'num_input_tokens_seen': 5043650560, 'completed': '65.23% (2_405 / 3_687)', 'remaining time': '11:45:03', 'throughput': '8691.14', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 14:57:26,079 >> {'loss': 0.2869, 'grad_norm': 15.946044921875, 'learning_rate': 3.6583825577108397e-07, 'epoch': 0.0016273393002441008, 'num_input_tokens_seen': 5045747712, 'completed': '65.26% (2_406 / 3_687)', 'remaining time': '11:31:27', 'throughput': '8938.09', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 14:57:55,973 >> {'loss': 0.544, 'grad_norm': 24.774717330932617, 'learning_rate': 3.654699953462494e-07, 'epoch': 0.001898562516951451, 'num_input_tokens_seen': 5047844864, 'completed': '65.28% (2_407 / 3_687)', 'remaining time': '11:23:18', 'throughput': '8769.31', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 14:58:25,739 >> {'loss': 0.34, 'grad_norm': 14.22808837890625, 'learning_rate': 3.651018834241406e-07, 'epoch': 0.0021697857336588013, 'num_input_tokens_seen': 5049942016, 'completed': '65.31% (2_408 / 3_687)', 'remaining time': '11:16:44', 'throughput': '8806.74', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 14:59:00,238 >> {'loss': 0.3746, 'grad_norm': 13.739176750183105, 'learning_rate': 3.6473392030100014e-07, 'epoch': 0.0024410089503661514, 'num_input_tokens_seen': 5052039168, 'completed': '65.34% (2_409 / 3_687)', 'remaining time': '11:22:43', 'throughput': '7598.49', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 14:59:31,662 >> {'loss': 0.4212, 'grad_norm': 16.19430160522461, 'learning_rate': 3.6436610627295074e-07, 'epoch': 0.0027122321670735015, 'num_input_tokens_seen': 5054136320, 'completed': '65.36% (2_410 / 3_687)', 'remaining time': '11:20:51', 'throughput': '8342.11', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 14:59:59,722 >> {'loss': 0.6517, 'grad_norm': 18.812156677246094, 'learning_rate': 3.639984416359949e-07, 'epoch': 0.0029834553837808516, 'num_input_tokens_seen': 5056233472, 'completed': '65.39% (2_411 / 3_687)', 'remaining time': '11:12:43', 'throughput': '9342.54', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 15:00:30,808 >> {'loss': 0.301, 'grad_norm': 10.88464069366455, 'learning_rate': 3.636309266860156e-07, 'epoch': 0.0032546786004882017, 'num_input_tokens_seen': 5058330624, 'completed': '65.42% (2_412 / 3_687)', 'remaining time': '11:11:13', 'throughput': '8432.72', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 15:01:00,624 >> {'loss': 0.5711, 'grad_norm': 17.772342681884766, 'learning_rate': 3.6326356171877483e-07, 'epoch': 0.003525901817195552, 'num_input_tokens_seen': 5060427776, 'completed': '65.45% (2_413 / 3_687)', 'remaining time': '11:07:48', 'throughput': '8792.11', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 15:01:32,736 >> {'loss': 0.3508, 'grad_norm': 18.594541549682617, 'learning_rate': 3.6289634702991343e-07, 'epoch': 0.003797125033902902, 'num_input_tokens_seen': 5062524928, 'completed': '65.47% (2_414 / 3_687)', 'remaining time': '11:08:17', 'throughput': '8163.46', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 15:02:04,847 >> {'loss': 0.2594, 'grad_norm': 12.95483112335205, 'learning_rate': 3.625292829149521e-07, 'epoch': 0.0040683482506102524, 'num_input_tokens_seen': 5064622080, 'completed': '65.50% (2_415 / 3_687)', 'remaining time': '11:08:37', 'throughput': '8163.72', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 15:02:34,568 >> {'loss': 0.7946, 'grad_norm': 23.838085174560547, 'learning_rate': 3.6216236966929015e-07, 'epoch': 0.0043395714673176026, 'num_input_tokens_seen': 5066719232, 'completed': '65.53% (2_416 / 3_687)', 'remaining time': '11:05:41', 'throughput': '8820.07', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 15:03:05,536 >> {'loss': 0.359, 'grad_norm': 17.340089797973633, 'learning_rate': 3.6179560758820527e-07, 'epoch': 0.004610794684024953, 'num_input_tokens_seen': 5068816384, 'completed': '65.55% (2_417 / 3_687)', 'remaining time': '11:04:36', 'throughput': '8464.94', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 15:03:35,512 >> {'loss': 0.5987, 'grad_norm': 30.172731399536133, 'learning_rate': 3.61428996966854e-07, 'epoch': 0.004882017900732303, 'num_input_tokens_seen': 5070913536, 'completed': '65.58% (2_418 / 3_687)', 'remaining time': '11:02:24', 'throughput': '8745.29', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 15:04:10,852 >> {'loss': 0.311, 'grad_norm': 16.280479431152344, 'learning_rate': 3.610625381002701e-07, 'epoch': 0.005153241117439653, 'num_input_tokens_seen': 5073010688, 'completed': '65.61% (2_419 / 3_687)', 'remaining time': '11:06:21', 'throughput': '7417.72', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 15:04:42,282 >> {'loss': 0.495, 'grad_norm': 17.15519142150879, 'learning_rate': 3.606962312833659e-07, 'epoch': 0.005424464334147003, 'num_input_tokens_seen': 5075107840, 'completed': '65.64% (2_420 / 3_687)', 'remaining time': '11:05:43', 'throughput': '8340.48', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 15:05:14,488 >> {'loss': 0.3901, 'grad_norm': 16.51264190673828, 'learning_rate': 3.603300768109311e-07, 'epoch': 0.005695687550854353, 'num_input_tokens_seen': 5077204992, 'completed': '65.66% (2_421 / 3_687)', 'remaining time': '11:05:53', 'throughput': '8139.58', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 15:05:44,685 >> {'loss': 0.6579, 'grad_norm': 22.195186614990234, 'learning_rate': 3.5996407497763305e-07, 'epoch': 0.005966910767561703, 'num_input_tokens_seen': 5079302144, 'completed': '65.69% (2_422 / 3_687)', 'remaining time': '11:04:03', 'throughput': '8681.23', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 15:06:15,336 >> {'loss': 0.3808, 'grad_norm': 14.603880882263184, 'learning_rate': 3.5959822607801617e-07, 'epoch': 0.006238133984269053, 'num_input_tokens_seen': 5081399296, 'completed': '65.72% (2_423 / 3_687)', 'remaining time': '11:02:45', 'throughput': '8552.52', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 15:06:46,312 >> {'loss': 0.5294, 'grad_norm': 19.53980255126953, 'learning_rate': 3.592325304065018e-07, 'epoch': 0.006509357200976403, 'num_input_tokens_seen': 5083496448, 'completed': '65.74% (2_424 / 3_687)', 'remaining time': '11:01:48', 'throughput': '8462.76', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 15:07:15,687 >> {'loss': 0.4601, 'grad_norm': 17.005908966064453, 'learning_rate': 3.588669882573875e-07, 'epoch': 0.0067805804176837535, 'num_input_tokens_seen': 5085593600, 'completed': '65.77% (2_425 / 3_687)', 'remaining time': '10:59:32', 'throughput': '8924.22', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 15:07:44,379 >> {'loss': 0.9515, 'grad_norm': 26.571182250976562, 'learning_rate': 3.5850159992484787e-07, 'epoch': 0.007051803634391104, 'num_input_tokens_seen': 5087690752, 'completed': '65.80% (2_426 / 3_687)', 'remaining time': '10:56:52', 'throughput': '9136.30', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 15:08:15,169 >> {'loss': 0.3284, 'grad_norm': 15.586816787719727, 'learning_rate': 3.581363657029336e-07, 'epoch': 0.007323026851098454, 'num_input_tokens_seen': 5089787904, 'completed': '65.83% (2_427 / 3_687)', 'remaining time': '10:55:59', 'throughput': '8513.97', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 15:08:47,836 >> {'loss': 0.1959, 'grad_norm': 9.308566093444824, 'learning_rate': 3.5777128588557126e-07, 'epoch': 0.007594250067805804, 'num_input_tokens_seen': 5091885056, 'completed': '65.85% (2_428 / 3_687)', 'remaining time': '10:56:32', 'throughput': '8024.66', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 15:09:18,729 >> {'loss': 0.6265, 'grad_norm': 18.024173736572266, 'learning_rate': 3.574063607665633e-07, 'epoch': 0.007865473284513154, 'num_input_tokens_seen': 5093982208, 'completed': '65.88% (2_429 / 3_687)', 'remaining time': '10:55:43', 'throughput': '8485.54', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 15:09:48,173 >> {'loss': 0.4991, 'grad_norm': 17.78046417236328, 'learning_rate': 3.570415906395873e-07, 'epoch': 0.008136696501220505, 'num_input_tokens_seen': 5096079360, 'completed': '65.91% (2_430 / 3_687)', 'remaining time': '10:53:55', 'throughput': '8903.13', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 15:10:18,288 >> {'loss': 0.455, 'grad_norm': 16.535097122192383, 'learning_rate': 3.5667697579819655e-07, 'epoch': 0.008407919717927854, 'num_input_tokens_seen': 5098176512, 'completed': '65.93% (2_431 / 3_687)', 'remaining time': '10:52:39', 'throughput': '8704.94', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 15:10:48,631 >> {'loss': 0.5008, 'grad_norm': 18.70627784729004, 'learning_rate': 3.563125165358193e-07, 'epoch': 0.008679142934635205, 'num_input_tokens_seen': 5100273664, 'completed': '65.96% (2_432 / 3_687)', 'remaining time': '10:51:36', 'throughput': '8639.31', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 15:11:17,922 >> {'loss': 0.4694, 'grad_norm': 17.156047821044922, 'learning_rate': 3.559482131457583e-07, 'epoch': 0.008950366151342554, 'num_input_tokens_seen': 5102370816, 'completed': '65.99% (2_433 / 3_687)', 'remaining time': '10:49:54', 'throughput': '8949.70', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 15:11:49,615 >> {'loss': 0.4304, 'grad_norm': 15.62370777130127, 'learning_rate': 3.5558406592119115e-07, 'epoch': 0.009221589368049905, 'num_input_tokens_seen': 5104467968, 'completed': '66.02% (2_434 / 3_687)', 'remaining time': '10:49:45', 'throughput': '8271.34', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 15:12:18,801 >> {'loss': 0.4884, 'grad_norm': 14.981012344360352, 'learning_rate': 3.552200751551697e-07, 'epoch': 0.009492812584757255, 'num_input_tokens_seen': 5106565120, 'completed': '66.04% (2_435 / 3_687)', 'remaining time': '10:48:04', 'throughput': '8981.90', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 15:12:48,695 >> {'loss': 0.4192, 'grad_norm': 16.954042434692383, 'learning_rate': 3.548562411406201e-07, 'epoch': 0.009764035801464606, 'num_input_tokens_seen': 5108662272, 'completed': '66.07% (2_436 / 3_687)', 'remaining time': '10:46:53', 'throughput': '8769.02', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 15:13:19,340 >> {'loss': 0.4403, 'grad_norm': 20.573305130004883, 'learning_rate': 3.544925641703413e-07, 'epoch': 0.010035259018171955, 'num_input_tokens_seen': 5110759424, 'completed': '66.10% (2_437 / 3_687)', 'remaining time': '10:46:09', 'throughput': '8554.38', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 15:13:47,923 >> {'loss': 0.6544, 'grad_norm': 22.3472957611084, 'learning_rate': 3.5412904453700754e-07, 'epoch': 0.010306482234879306, 'num_input_tokens_seen': 5112856576, 'completed': '66.12% (2_438 / 3_687)', 'remaining time': '10:44:18', 'throughput': '9171.13', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 15:14:15,963 >> {'loss': 0.6427, 'grad_norm': 33.12949752807617, 'learning_rate': 3.537656825331653e-07, 'epoch': 0.010577705451586655, 'num_input_tokens_seen': 5114953728, 'completed': '66.15% (2_439 / 3_687)', 'remaining time': '10:42:14', 'throughput': '9348.85', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 15:14:49,229 >> {'loss': 0.501, 'grad_norm': 18.552413940429688, 'learning_rate': 3.534024784512345e-07, 'epoch': 0.010848928668294006, 'num_input_tokens_seen': 5117050880, 'completed': '66.18% (2_440 / 3_687)', 'remaining time': '10:42:58', 'throughput': '7880.35', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 15:15:20,514 >> {'loss': 0.3836, 'grad_norm': 15.270557403564453, 'learning_rate': 3.5303943258350813e-07, 'epoch': 0.011120151885001357, 'num_input_tokens_seen': 5119148032, 'completed': '66.21% (2_441 / 3_687)', 'remaining time': '10:42:37', 'throughput': '8379.17', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 15:15:51,724 >> {'loss': 0.3691, 'grad_norm': 32.084510803222656, 'learning_rate': 3.526765452221512e-07, 'epoch': 0.011391375101708706, 'num_input_tokens_seen': 5121245184, 'completed': '66.23% (2_442 / 3_687)', 'remaining time': '10:42:14', 'throughput': '8399.50', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 15:16:24,072 >> {'loss': 0.4435, 'grad_norm': 15.427001953125, 'learning_rate': 3.523138166592021e-07, 'epoch': 0.011662598318416057, 'num_input_tokens_seen': 5123342336, 'completed': '66.26% (2_443 / 3_687)', 'remaining time': '10:42:24', 'throughput': '8103.71', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 15:16:54,026 >> {'loss': 0.725, 'grad_norm': 21.460124969482422, 'learning_rate': 3.5195124718657075e-07, 'epoch': 0.011933821535123406, 'num_input_tokens_seen': 5125439488, 'completed': '66.29% (2_444 / 3_687)', 'remaining time': '10:41:24', 'throughput': '8751.49', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 15:17:23,822 >> {'loss': 0.2977, 'grad_norm': 15.456875801086426, 'learning_rate': 3.5158883709603946e-07, 'epoch': 0.012205044751830757, 'num_input_tokens_seen': 5127536640, 'completed': '66.31% (2_445 / 3_687)', 'remaining time': '10:40:20', 'throughput': '8798.15', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 15:17:58,011 >> {'loss': 0.5319, 'grad_norm': 16.195775985717773, 'learning_rate': 3.5122658667926177e-07, 'epoch': 0.012476267968538107, 'num_input_tokens_seen': 5129633792, 'completed': '66.34% (2_446 / 3_687)', 'remaining time': '10:41:17', 'throughput': '7667.44', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 15:18:27,105 >> {'loss': 0.5982, 'grad_norm': 16.876602172851562, 'learning_rate': 3.5086449622776346e-07, 'epoch': 0.012747491185245458, 'num_input_tokens_seen': 5131730944, 'completed': '66.37% (2_447 / 3_687)', 'remaining time': '10:39:56', 'throughput': '9010.38', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 15:18:57,681 >> {'loss': 0.4243, 'grad_norm': 16.981779098510742, 'learning_rate': 3.505025660329408e-07, 'epoch': 0.013018714401952807, 'num_input_tokens_seen': 5133828096, 'completed': '66.40% (2_448 / 3_687)', 'remaining time': '10:39:15', 'throughput': '8573.54', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 15:19:28,568 >> {'loss': 0.3373, 'grad_norm': 13.226367950439453, 'learning_rate': 3.5014079638606164e-07, 'epoch': 0.013289937618660158, 'num_input_tokens_seen': 5135925248, 'completed': '66.42% (2_449 / 3_687)', 'remaining time': '10:38:42', 'throughput': '8487.20', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 15:19:57,176 >> {'loss': 0.3153, 'grad_norm': 14.811175346374512, 'learning_rate': 3.497791875782643e-07, 'epoch': 0.013561160835367507, 'num_input_tokens_seen': 5138022400, 'completed': '66.45% (2_450 / 3_687)', 'remaining time': '10:37:13', 'throughput': '9163.01', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 15:20:29,107 >> {'loss': 0.4019, 'grad_norm': 17.1142578125, 'learning_rate': 3.4941773990055777e-07, 'epoch': 0.013832384052074858, 'num_input_tokens_seen': 5140119552, 'completed': '66.48% (2_451 / 3_687)', 'remaining time': '10:37:07', 'throughput': '8209.89', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 15:20:58,933 >> {'loss': 0.7408, 'grad_norm': 24.696069717407227, 'learning_rate': 3.490564536438215e-07, 'epoch': 0.014103607268782207, 'num_input_tokens_seen': 5142216704, 'completed': '66.50% (2_452 / 3_687)', 'remaining time': '10:36:10', 'throughput': '8788.97', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 15:21:29,219 >> {'loss': 0.4626, 'grad_norm': 15.740670204162598, 'learning_rate': 3.4869532909880485e-07, 'epoch': 0.014374830485489558, 'num_input_tokens_seen': 5144313856, 'completed': '66.53% (2_453 / 3_687)', 'remaining time': '10:35:24', 'throughput': '8655.80', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 15:21:59,181 >> {'loss': 0.3852, 'grad_norm': 17.356019973754883, 'learning_rate': 3.483343665561271e-07, 'epoch': 0.014646053702196907, 'num_input_tokens_seen': 5146411008, 'completed': '66.56% (2_454 / 3_687)', 'remaining time': '10:34:32', 'throughput': '8749.03', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 15:22:31,655 >> {'loss': 0.5229, 'grad_norm': 19.49802589416504, 'learning_rate': 3.479735663062773e-07, 'epoch': 0.014917276918904258, 'num_input_tokens_seen': 5148508160, 'completed': '66.59% (2_455 / 3_687)', 'remaining time': '10:34:37', 'throughput': '8072.48', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 15:23:01,569 >> {'loss': 0.3333, 'grad_norm': 11.227045059204102, 'learning_rate': 3.4761292863961354e-07, 'epoch': 0.015188500135611608, 'num_input_tokens_seen': 5150605312, 'completed': '66.61% (2_456 / 3_687)', 'remaining time': '10:33:44', 'throughput': '8763.10', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 15:23:31,993 >> {'loss': 0.4247, 'grad_norm': 17.022945404052734, 'learning_rate': 3.4725245384636347e-07, 'epoch': 0.015459723352318959, 'num_input_tokens_seen': 5152702464, 'completed': '66.64% (2_457 / 3_687)', 'remaining time': '10:33:03', 'throughput': '8616.41', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 15:24:01,898 >> {'loss': 0.2605, 'grad_norm': 12.745018005371094, 'learning_rate': 3.4689214221662364e-07, 'epoch': 0.015730946569026308, 'num_input_tokens_seen': 5154799616, 'completed': '66.67% (2_458 / 3_687)', 'remaining time': '10:32:12', 'throughput': '8765.92', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 15:24:33,027 >> {'loss': 0.5055, 'grad_norm': 16.132686614990234, 'learning_rate': 3.465319940403587e-07, 'epoch': 0.01600216978573366, 'num_input_tokens_seen': 5156896768, 'completed': '66.69% (2_459 / 3_687)', 'remaining time': '10:31:47', 'throughput': '8421.26', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 15:25:02,591 >> {'loss': 0.3678, 'grad_norm': 15.20864486694336, 'learning_rate': 3.4617200960740247e-07, 'epoch': 0.01627339300244101, 'num_input_tokens_seen': 5158993920, 'completed': '66.72% (2_460 / 3_687)', 'remaining time': '10:30:49', 'throughput': '8867.05', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 15:25:34,521 >> {'loss': 0.3402, 'grad_norm': 12.260754585266113, 'learning_rate': 3.4581218920745663e-07, 'epoch': 0.01654461621914836, 'num_input_tokens_seen': 5161091072, 'completed': '66.75% (2_461 / 3_687)', 'remaining time': '10:30:40', 'throughput': '8209.91', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 15:26:04,071 >> {'loss': 0.3149, 'grad_norm': 13.886555671691895, 'learning_rate': 3.454525331300908e-07, 'epoch': 0.016815839435855708, 'num_input_tokens_seen': 5163188224, 'completed': '66.78% (2_462 / 3_687)', 'remaining time': '10:29:43', 'throughput': '8871.26', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 15:26:35,288 >> {'loss': 0.4094, 'grad_norm': 22.19728660583496, 'learning_rate': 3.450930416647429e-07, 'epoch': 0.01708706265256306, 'num_input_tokens_seen': 5165285376, 'completed': '66.80% (2_463 / 3_687)', 'remaining time': '10:29:19', 'throughput': '8397.31', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 15:27:05,006 >> {'loss': 0.4385, 'grad_norm': 14.57243824005127, 'learning_rate': 3.4473371510071795e-07, 'epoch': 0.01735828586927041, 'num_input_tokens_seen': 5167382528, 'completed': '66.83% (2_464 / 3_687)', 'remaining time': '10:28:27', 'throughput': '8821.10', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 15:27:36,034 >> {'loss': 0.5753, 'grad_norm': 20.04316520690918, 'learning_rate': 3.4437455372718795e-07, 'epoch': 0.01762950908597776, 'num_input_tokens_seen': 5169479680, 'completed': '66.86% (2_465 / 3_687)', 'remaining time': '10:28:00', 'throughput': '8448.62', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 15:28:08,889 >> {'loss': 0.3834, 'grad_norm': 18.932300567626953, 'learning_rate': 3.440155578331925e-07, 'epoch': 0.01790073230268511, 'num_input_tokens_seen': 5171576832, 'completed': '66.88% (2_466 / 3_687)', 'remaining time': '10:28:06', 'throughput': '7978.91', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 15:28:36,506 >> {'loss': 0.3425, 'grad_norm': 15.249947547912598, 'learning_rate': 3.4365672770763783e-07, 'epoch': 0.01817195551939246, 'num_input_tokens_seen': 5173673984, 'completed': '66.91% (2_467 / 3_687)', 'remaining time': '10:26:36', 'throughput': '9492.15', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 15:29:08,955 >> {'loss': 0.3937, 'grad_norm': 16.69132423400879, 'learning_rate': 3.432980636392967e-07, 'epoch': 0.01844317873609981, 'num_input_tokens_seen': 5175771136, 'completed': '66.94% (2_468 / 3_687)', 'remaining time': '10:26:35', 'throughput': '8078.55', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 15:29:38,880 >> {'loss': 0.3516, 'grad_norm': 14.121512413024902, 'learning_rate': 3.429395659168084e-07, 'epoch': 0.01871440195280716, 'num_input_tokens_seen': 5177868288, 'completed': '66.97% (2_469 / 3_687)', 'remaining time': '10:25:48', 'throughput': '8760.24', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 15:30:06,994 >> {'loss': 0.5579, 'grad_norm': 19.293235778808594, 'learning_rate': 3.425812348286782e-07, 'epoch': 0.01898562516951451, 'num_input_tokens_seen': 5179965440, 'completed': '66.99% (2_470 / 3_687)', 'remaining time': '10:24:30', 'throughput': '9324.02', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 15:30:34,632 >> {'loss': 0.8517, 'grad_norm': 26.16200065612793, 'learning_rate': 3.422230706632774e-07, 'epoch': 0.01925684838622186, 'num_input_tokens_seen': 5182062592, 'completed': '67.02% (2_471 / 3_687)', 'remaining time': '10:23:05', 'throughput': '9485.00', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 15:31:04,208 >> {'loss': 0.4796, 'grad_norm': 16.774105072021484, 'learning_rate': 3.418650737088427e-07, 'epoch': 0.01952807160292921, 'num_input_tokens_seen': 5184159744, 'completed': '67.05% (2_472 / 3_687)', 'remaining time': '10:22:15', 'throughput': '8863.54', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 15:31:35,457 >> {'loss': 0.3243, 'grad_norm': 16.8173885345459, 'learning_rate': 3.415072442534767e-07, 'epoch': 0.019799294819636562, 'num_input_tokens_seen': 5186256896, 'completed': '67.07% (2_473 / 3_687)', 'remaining time': '10:21:52', 'throughput': '8388.73', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 15:32:06,739 >> {'loss': 0.3466, 'grad_norm': 14.54384994506836, 'learning_rate': 3.411495825851467e-07, 'epoch': 0.02007051803634391, 'num_input_tokens_seen': 5188354048, 'completed': '67.10% (2_474 / 3_687)', 'remaining time': '10:21:31', 'throughput': '8380.18', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 15:32:36,423 >> {'loss': 0.3709, 'grad_norm': 16.708059310913086, 'learning_rate': 3.4079208899168545e-07, 'epoch': 0.02034174125305126, 'num_input_tokens_seen': 5190451200, 'completed': '67.13% (2_475 / 3_687)', 'remaining time': '10:20:43', 'throughput': '8830.95', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 15:33:07,157 >> {'loss': 0.4063, 'grad_norm': 18.377099990844727, 'learning_rate': 3.404347637607899e-07, 'epoch': 0.02061296446975861, 'num_input_tokens_seen': 5192548352, 'completed': '67.15% (2_476 / 3_687)', 'remaining time': '10:20:12', 'throughput': '8529.54', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 15:33:36,670 >> {'loss': 0.4975, 'grad_norm': 21.520936965942383, 'learning_rate': 3.400776071800219e-07, 'epoch': 0.020884187686465962, 'num_input_tokens_seen': 5194645504, 'completed': '67.18% (2_477 / 3_687)', 'remaining time': '10:19:22', 'throughput': '8882.37', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 15:34:06,501 >> {'loss': 0.2697, 'grad_norm': 12.55679702758789, 'learning_rate': 3.3972061953680734e-07, 'epoch': 0.02115541090317331, 'num_input_tokens_seen': 5196742656, 'completed': '67.21% (2_478 / 3_687)', 'remaining time': '10:18:38', 'throughput': '8787.74', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 15:34:39,660 >> {'loss': 0.4419, 'grad_norm': 16.432504653930664, 'learning_rate': 3.3936380111843666e-07, 'epoch': 0.02142663411988066, 'num_input_tokens_seen': 5198839808, 'completed': '67.24% (2_479 / 3_687)', 'remaining time': '10:18:45', 'throughput': '7905.55', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 15:35:11,771 >> {'loss': 0.3704, 'grad_norm': 12.978721618652344, 'learning_rate': 3.390071522120635e-07, 'epoch': 0.021697857336588012, 'num_input_tokens_seen': 5200936960, 'completed': '67.26% (2_480 / 3_687)', 'remaining time': '10:18:35', 'throughput': '8163.77', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 15:35:43,392 >> {'loss': 0.3597, 'grad_norm': 13.821551322937012, 'learning_rate': 3.3865067310470554e-07, 'epoch': 0.021969080553295363, 'num_input_tokens_seen': 5203034112, 'completed': '67.29% (2_481 / 3_687)', 'remaining time': '10:18:17', 'throughput': '8290.04', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 15:36:12,516 >> {'loss': 0.3515, 'grad_norm': 13.253721237182617, 'learning_rate': 3.3829436408324316e-07, 'epoch': 0.022240303770002714, 'num_input_tokens_seen': 5205131264, 'completed': '67.32% (2_482 / 3_687)', 'remaining time': '10:17:22', 'throughput': '9001.01', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 15:36:44,328 >> {'loss': 0.4505, 'grad_norm': 15.398584365844727, 'learning_rate': 3.3793822543442074e-07, 'epoch': 0.02251152698671006, 'num_input_tokens_seen': 5207228416, 'completed': '67.34% (2_483 / 3_687)', 'remaining time': '10:17:07', 'throughput': '8240.31', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 15:37:16,285 >> {'loss': 0.4072, 'grad_norm': 17.346776962280273, 'learning_rate': 3.3758225744484483e-07, 'epoch': 0.022782750203417412, 'num_input_tokens_seen': 5209325568, 'completed': '67.37% (2_484 / 3_687)', 'remaining time': '10:16:54', 'throughput': '8203.24', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 15:37:48,188 >> {'loss': 0.4183, 'grad_norm': 14.590849876403809, 'learning_rate': 3.372264604009851e-07, 'epoch': 0.023053973420124763, 'num_input_tokens_seen': 5211422720, 'completed': '67.40% (2_485 / 3_687)', 'remaining time': '10:16:39', 'throughput': '8216.69', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 15:38:17,177 >> {'loss': 0.4176, 'grad_norm': 16.34534454345703, 'learning_rate': 3.3687083458917344e-07, 'epoch': 0.023325196636832114, 'num_input_tokens_seen': 5213519872, 'completed': '67.43% (2_486 / 3_687)', 'remaining time': '10:15:43', 'throughput': '9043.05', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 15:38:48,130 >> {'loss': 0.3908, 'grad_norm': 15.394360542297363, 'learning_rate': 3.3651538029560377e-07, 'epoch': 0.023596419853539462, 'num_input_tokens_seen': 5215617024, 'completed': '67.45% (2_487 / 3_687)', 'remaining time': '10:15:15', 'throughput': '8469.07', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 15:39:16,722 >> {'loss': 0.4385, 'grad_norm': 15.60722541809082, 'learning_rate': 3.361600978063325e-07, 'epoch': 0.023867643070246813, 'num_input_tokens_seen': 5217714176, 'completed': '67.48% (2_488 / 3_687)', 'remaining time': '10:14:15', 'throughput': '9168.53', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 15:39:46,385 >> {'loss': 0.4593, 'grad_norm': 18.028072357177734, 'learning_rate': 3.358049874072771e-07, 'epoch': 0.024138866286954164, 'num_input_tokens_seen': 5219811328, 'completed': '67.51% (2_489 / 3_687)', 'remaining time': '10:13:29', 'throughput': '8837.25', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 15:40:18,046 >> {'loss': 0.4059, 'grad_norm': 18.258270263671875, 'learning_rate': 3.3545004938421734e-07, 'epoch': 0.024410089503661515, 'num_input_tokens_seen': 5221908480, 'completed': '67.53% (2_490 / 3_687)', 'remaining time': '10:13:11', 'throughput': '8279.81', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 15:40:45,496 >> {'loss': 0.7823, 'grad_norm': 21.254680633544922, 'learning_rate': 3.3509528402279357e-07, 'epoch': 0.024681312720368862, 'num_input_tokens_seen': 5224005632, 'completed': '67.56% (2_491 / 3_687)', 'remaining time': '10:11:57', 'throughput': '9549.78', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 15:41:17,066 >> {'loss': 0.9546, 'grad_norm': 37.15273666381836, 'learning_rate': 3.347406916085074e-07, 'epoch': 0.024952535937076213, 'num_input_tokens_seen': 5226102784, 'completed': '67.59% (2_492 / 3_687)', 'remaining time': '10:11:38', 'throughput': '8303.73', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 15:41:48,190 >> {'loss': 0.7096, 'grad_norm': 19.634872436523438, 'learning_rate': 3.3438627242672164e-07, 'epoch': 0.025223759153783564, 'num_input_tokens_seen': 5228199936, 'completed': '67.62% (2_493 / 3_687)', 'remaining time': '10:11:12', 'throughput': '8422.34', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 15:42:19,435 >> {'loss': 0.4221, 'grad_norm': 18.12146759033203, 'learning_rate': 3.3403202676265875e-07, 'epoch': 0.025494982370490915, 'num_input_tokens_seen': 5230297088, 'completed': '67.64% (2_494 / 3_687)', 'remaining time': '10:10:48', 'throughput': '8390.26', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 15:42:52,937 >> {'loss': 0.2631, 'grad_norm': 11.500085830688477, 'learning_rate': 3.336779549014026e-07, 'epoch': 0.025766205587198263, 'num_input_tokens_seen': 5232394240, 'completed': '67.67% (2_495 / 3_687)', 'remaining time': '10:10:53', 'throughput': '7824.50', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 15:43:22,285 >> {'loss': 0.3061, 'grad_norm': 13.041985511779785, 'learning_rate': 3.333240571278968e-07, 'epoch': 0.026037428803905614, 'num_input_tokens_seen': 5234491392, 'completed': '67.70% (2_496 / 3_687)', 'remaining time': '10:10:04', 'throughput': '8932.22', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 15:43:53,848 >> {'loss': 0.2412, 'grad_norm': 9.483246803283691, 'learning_rate': 3.3297033372694473e-07, 'epoch': 0.026308652020612965, 'num_input_tokens_seen': 5236588544, 'completed': '67.72% (2_497 / 3_687)', 'remaining time': '10:09:44', 'throughput': '8305.46', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 15:44:20,471 >> {'loss': 0.8186, 'grad_norm': 20.214548110961914, 'learning_rate': 3.3261678498320954e-07, 'epoch': 0.026579875237320316, 'num_input_tokens_seen': 5238685696, 'completed': '67.75% (2_498 / 3_687)', 'remaining time': '10:08:23', 'throughput': '9846.55', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 15:44:47,452 >> {'loss': 0.5537, 'grad_norm': 16.735645294189453, 'learning_rate': 3.3226341118121367e-07, 'epoch': 0.026851098454027666, 'num_input_tokens_seen': 5240782848, 'completed': '67.78% (2_499 / 3_687)', 'remaining time': '10:07:08', 'throughput': '9716.01', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 15:45:18,670 >> {'loss': 0.3094, 'grad_norm': 11.246064186096191, 'learning_rate': 3.319102126053389e-07, 'epoch': 0.027122321670735014, 'num_input_tokens_seen': 5242880000, 'completed': '67.81% (2_500 / 3_687)', 'remaining time': '10:06:44', 'throughput': '8397.08', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 15:45:51,530 >> {'loss': 0.2749, 'grad_norm': 16.4909610748291, 'learning_rate': 3.315571895398261e-07, 'epoch': 0.027393544887442365, 'num_input_tokens_seen': 5244977152, 'completed': '67.83% (2_501 / 3_687)', 'remaining time': '10:06:39', 'throughput': '7977.61', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 15:46:22,796 >> {'loss': 0.3926, 'grad_norm': 14.262845993041992, 'learning_rate': 3.312043422687749e-07, 'epoch': 0.027664768104149716, 'num_input_tokens_seen': 5247074304, 'completed': '67.86% (2_502 / 3_687)', 'remaining time': '10:06:15', 'throughput': '8384.47', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 15:46:55,572 >> {'loss': 0.2012, 'grad_norm': 11.372760772705078, 'learning_rate': 3.3085167107614297e-07, 'epoch': 0.027935991320857067, 'num_input_tokens_seen': 5249171456, 'completed': '67.89% (2_503 / 3_687)', 'remaining time': '10:06:08', 'throughput': '7997.86', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 15:47:24,427 >> {'loss': 0.7148, 'grad_norm': 21.803176879882812, 'learning_rate': 3.3049917624574737e-07, 'epoch': 0.028207214537564414, 'num_input_tokens_seen': 5251268608, 'completed': '67.91% (2_504 / 3_687)', 'remaining time': '10:05:16', 'throughput': '9085.02', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 15:47:56,443 >> {'loss': 0.3626, 'grad_norm': 14.963140487670898, 'learning_rate': 3.301468580612619e-07, 'epoch': 0.028478437754271765, 'num_input_tokens_seen': 5253365760, 'completed': '67.94% (2_505 / 3_687)', 'remaining time': '10:05:00', 'throughput': '8187.77', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 15:48:27,253 >> {'loss': 0.325, 'grad_norm': 16.734357833862305, 'learning_rate': 3.2979471680621903e-07, 'epoch': 0.028749660970979116, 'num_input_tokens_seen': 5255462912, 'completed': '67.97% (2_506 / 3_687)', 'remaining time': '10:04:31', 'throughput': '8508.45', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 15:48:55,714 >> {'loss': 0.5537, 'grad_norm': 18.527103424072266, 'learning_rate': 3.2944275276400857e-07, 'epoch': 0.029020884187686467, 'num_input_tokens_seen': 5257560064, 'completed': '68.00% (2_507 / 3_687)', 'remaining time': '10:03:35', 'throughput': '9210.87', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 15:49:26,266 >> {'loss': 0.4739, 'grad_norm': 17.63797950744629, 'learning_rate': 3.290909662178779e-07, 'epoch': 0.029292107404393815, 'num_input_tokens_seen': 5259657216, 'completed': '68.02% (2_508 / 3_687)', 'remaining time': '10:03:03', 'throughput': '8580.18', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 15:49:56,659 >> {'loss': 0.1794, 'grad_norm': 11.94525146484375, 'learning_rate': 3.2873935745093145e-07, 'epoch': 0.029563330621101166, 'num_input_tokens_seen': 5261754368, 'completed': '68.05% (2_509 / 3_687)', 'remaining time': '10:02:29', 'throughput': '8625.19', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 15:50:28,804 >> {'loss': 0.3807, 'grad_norm': 15.735551834106445, 'learning_rate': 3.283879267461305e-07, 'epoch': 0.029834553837808517, 'num_input_tokens_seen': 5263851520, 'completed': '68.08% (2_510 / 3_687)', 'remaining time': '10:02:14', 'throughput': '8155.10', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 15:50:58,987 >> {'loss': 0.2682, 'grad_norm': 14.347485542297363, 'learning_rate': 3.280366743862931e-07, 'epoch': 0.030105777054515868, 'num_input_tokens_seen': 5265948672, 'completed': '68.10% (2_511 / 3_687)', 'remaining time': '10:01:38', 'throughput': '8685.13', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 15:51:33,129 >> {'loss': 0.4171, 'grad_norm': 16.711332321166992, 'learning_rate': 3.276856006540939e-07, 'epoch': 0.030377000271223215, 'num_input_tokens_seen': 5268045824, 'completed': '68.13% (2_512 / 3_687)', 'remaining time': '10:01:43', 'throughput': '7677.88', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 15:52:04,025 >> {'loss': 0.7826, 'grad_norm': 22.094377517700195, 'learning_rate': 3.2733470583206357e-07, 'epoch': 0.030648223487930566, 'num_input_tokens_seen': 5270142976, 'completed': '68.16% (2_513 / 3_687)', 'remaining time': '10:01:14', 'throughput': '8484.85', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 15:52:34,386 >> {'loss': 0.6522, 'grad_norm': 24.63713264465332, 'learning_rate': 3.2698399020258895e-07, 'epoch': 0.030919446704637917, 'num_input_tokens_seen': 5272240128, 'completed': '68.19% (2_514 / 3_687)', 'remaining time': '10:00:40', 'throughput': '8634.10', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 15:53:05,024 >> {'loss': 0.3224, 'grad_norm': 14.236292839050293, 'learning_rate': 3.266334540479128e-07, 'epoch': 0.031190669921345268, 'num_input_tokens_seen': 5274337280, 'completed': '68.21% (2_515 / 3_687)', 'remaining time': '10:00:08', 'throughput': '8556.35', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 15:53:35,004 >> {'loss': 0.4438, 'grad_norm': 17.167448043823242, 'learning_rate': 3.262830976501329e-07, 'epoch': 0.031461893138052616, 'num_input_tokens_seen': 5276434432, 'completed': '68.24% (2_516 / 3_687)', 'remaining time': '9:59:30', 'throughput': '8743.83', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 15:54:07,321 >> {'loss': 0.3641, 'grad_norm': 14.156777381896973, 'learning_rate': 3.2593292129120295e-07, 'epoch': 0.03173311635475997, 'num_input_tokens_seen': 5278531584, 'completed': '68.27% (2_517 / 3_687)', 'remaining time': '9:59:15', 'throughput': '8111.71', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 15:54:36,172 >> {'loss': 0.4017, 'grad_norm': 16.016916275024414, 'learning_rate': 3.2558292525293156e-07, 'epoch': 0.03200433957146732, 'num_input_tokens_seen': 5280628736, 'completed': '68.29% (2_518 / 3_687)', 'remaining time': '9:58:26', 'throughput': '9086.20', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 15:55:07,397 >> {'loss': 0.4632, 'grad_norm': 17.823566436767578, 'learning_rate': 3.2523310981698213e-07, 'epoch': 0.032275562788174665, 'num_input_tokens_seen': 5282725888, 'completed': '68.32% (2_519 / 3_687)', 'remaining time': '9:58:00', 'throughput': '8395.19', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 15:55:36,886 >> {'loss': 0.6673, 'grad_norm': 21.00741958618164, 'learning_rate': 3.24883475264873e-07, 'epoch': 0.03254678600488202, 'num_input_tokens_seen': 5284823040, 'completed': '68.35% (2_520 / 3_687)', 'remaining time': '9:57:17', 'throughput': '8889.63', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 15:56:07,967 >> {'loss': 0.4991, 'grad_norm': 14.62010669708252, 'learning_rate': 3.2453402187797684e-07, 'epoch': 0.03281800922158937, 'num_input_tokens_seen': 5286920192, 'completed': '68.38% (2_521 / 3_687)', 'remaining time': '9:56:50', 'throughput': '8434.15', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 15:56:38,269 >> {'loss': 0.4899, 'grad_norm': 17.261287689208984, 'learning_rate': 3.241847499375201e-07, 'epoch': 0.03308923243829672, 'num_input_tokens_seen': 5289017344, 'completed': '68.40% (2_522 / 3_687)', 'remaining time': '9:56:16', 'throughput': '8650.99', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 15:57:07,140 >> {'loss': 0.2637, 'grad_norm': 14.625133514404297, 'learning_rate': 3.238356597245837e-07, 'epoch': 0.03336045565500407, 'num_input_tokens_seen': 5291114496, 'completed': '68.43% (2_523 / 3_687)', 'remaining time': '9:55:27', 'throughput': '9080.09', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 15:57:36,373 >> {'loss': 0.4681, 'grad_norm': 17.17628288269043, 'learning_rate': 3.2348675152010217e-07, 'epoch': 0.033631678871711417, 'num_input_tokens_seen': 5293211648, 'completed': '68.46% (2_524 / 3_687)', 'remaining time': '9:54:43', 'throughput': '8967.27', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 15:58:07,870 >> {'loss': 0.5437, 'grad_norm': 21.153200149536133, 'learning_rate': 3.2313802560486353e-07, 'epoch': 0.03390290208841877, 'num_input_tokens_seen': 5295308800, 'completed': '68.48% (2_525 / 3_687)', 'remaining time': '9:54:20', 'throughput': '8322.85', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 15:58:38,746 >> {'loss': 0.607, 'grad_norm': 21.99086570739746, 'learning_rate': 3.2278948225950916e-07, 'epoch': 0.03417412530512612, 'num_input_tokens_seen': 5297405952, 'completed': '68.51% (2_526 / 3_687)', 'remaining time': '9:53:51', 'throughput': '8490.18', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 15:59:08,957 >> {'loss': 0.4772, 'grad_norm': 16.050336837768555, 'learning_rate': 3.2244112176453343e-07, 'epoch': 0.034445348521833466, 'num_input_tokens_seen': 5299503104, 'completed': '68.54% (2_527 / 3_687)', 'remaining time': '9:53:16', 'throughput': '8677.02', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 15:59:39,223 >> {'loss': 0.6767, 'grad_norm': 19.21502685546875, 'learning_rate': 3.2209294440028366e-07, 'epoch': 0.03471657173854082, 'num_input_tokens_seen': 5301600256, 'completed': '68.57% (2_528 / 3_687)', 'remaining time': '9:52:41', 'throughput': '8661.51', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 16:00:12,247 >> {'loss': 0.3405, 'grad_norm': 14.736310958862305, 'learning_rate': 3.2174495044695973e-07, 'epoch': 0.03498779495524817, 'num_input_tokens_seen': 5303697408, 'completed': '68.59% (2_529 / 3_687)', 'remaining time': '9:52:32', 'throughput': '7937.81', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 16:00:44,055 >> {'loss': 0.4872, 'grad_norm': 17.85927391052246, 'learning_rate': 3.2139714018461396e-07, 'epoch': 0.03525901817195552, 'num_input_tokens_seen': 5305794560, 'completed': '68.62% (2_530 / 3_687)', 'remaining time': '9:52:11', 'throughput': '8241.67', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 16:01:14,830 >> {'loss': 0.5331, 'grad_norm': 16.15865707397461, 'learning_rate': 3.2104951389315073e-07, 'epoch': 0.03553024138866287, 'num_input_tokens_seen': 5307891712, 'completed': '68.65% (2_531 / 3_687)', 'remaining time': '9:51:41', 'throughput': '8517.84', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 16:01:43,240 >> {'loss': 0.6059, 'grad_norm': 21.816003799438477, 'learning_rate': 3.207020718523266e-07, 'epoch': 0.03580146460537022, 'num_input_tokens_seen': 5309988864, 'completed': '68.67% (2_532 / 3_687)', 'remaining time': '9:50:50', 'throughput': '9227.23', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 16:02:14,090 >> {'loss': 0.3919, 'grad_norm': 15.507525444030762, 'learning_rate': 3.2035481434174966e-07, 'epoch': 0.03607268782207757, 'num_input_tokens_seen': 5312086016, 'completed': '68.70% (2_533 / 3_687)', 'remaining time': '9:50:21', 'throughput': '8497.39', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 16:02:43,737 >> {'loss': 0.4488, 'grad_norm': 15.8746919631958, 'learning_rate': 3.200077416408794e-07, 'epoch': 0.03634391103878492, 'num_input_tokens_seen': 5314183168, 'completed': '68.73% (2_534 / 3_687)', 'remaining time': '9:49:41', 'throughput': '8842.23', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 16:03:13,383 >> {'loss': 0.627, 'grad_norm': 20.78430938720703, 'learning_rate': 3.196608540290266e-07, 'epoch': 0.03661513425549227, 'num_input_tokens_seen': 5316280320, 'completed': '68.76% (2_535 / 3_687)', 'remaining time': '9:49:01', 'throughput': '8842.53', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 16:03:45,329 >> {'loss': 0.5527, 'grad_norm': 21.117902755737305, 'learning_rate': 3.193141517853536e-07, 'epoch': 0.03688635747219962, 'num_input_tokens_seen': 5318377472, 'completed': '68.78% (2_536 / 3_687)', 'remaining time': '9:48:41', 'throughput': '8205.90', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 16:04:16,028 >> {'loss': 0.4613, 'grad_norm': 14.952088356018066, 'learning_rate': 3.1896763518887305e-07, 'epoch': 0.03715758068890697, 'num_input_tokens_seen': 5320474624, 'completed': '68.81% (2_537 / 3_687)', 'remaining time': '9:48:11', 'throughput': '8538.98', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 16:04:43,100 >> {'loss': 0.8986, 'grad_norm': 25.334875106811523, 'learning_rate': 3.186213045184484e-07, 'epoch': 0.03742880390561432, 'num_input_tokens_seen': 5322571776, 'completed': '68.84% (2_538 / 3_687)', 'remaining time': '9:47:10', 'throughput': '9683.30', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 16:05:16,282 >> {'loss': 0.3267, 'grad_norm': 12.866583824157715, 'learning_rate': 3.1827516005279306e-07, 'epoch': 0.03770002712232167, 'num_input_tokens_seen': 5324668928, 'completed': '68.86% (2_539 / 3_687)', 'remaining time': '9:47:00', 'throughput': '7900.27', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 16:05:46,997 >> {'loss': 0.3742, 'grad_norm': 14.773965835571289, 'learning_rate': 3.1792920207047114e-07, 'epoch': 0.03797125033902902, 'num_input_tokens_seen': 5326766080, 'completed': '68.89% (2_540 / 3_687)', 'remaining time': '9:46:30', 'throughput': '8534.74', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 16:06:19,421 >> {'loss': 0.4086, 'grad_norm': 17.13579559326172, 'learning_rate': 3.175834308498964e-07, 'epoch': 0.03824247355573637, 'num_input_tokens_seen': 5328863232, 'completed': '68.92% (2_541 / 3_687)', 'remaining time': '9:46:13', 'throughput': '8084.78', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 16:06:49,500 >> {'loss': 0.3569, 'grad_norm': 16.100215911865234, 'learning_rate': 3.172378466693325e-07, 'epoch': 0.03851369677244372, 'num_input_tokens_seen': 5330960384, 'completed': '68.94% (2_542 / 3_687)', 'remaining time': '9:45:38', 'throughput': '8715.31', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 16:07:17,444 >> {'loss': 0.9546, 'grad_norm': 22.463777542114258, 'learning_rate': 3.168924498068923e-07, 'epoch': 0.038784919989151075, 'num_input_tokens_seen': 5333057536, 'completed': '68.97% (2_543 / 3_687)', 'remaining time': '9:44:45', 'throughput': '9381.06', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 16:07:48,533 >> {'loss': 0.4266, 'grad_norm': 17.804658889770508, 'learning_rate': 3.1654724054053805e-07, 'epoch': 0.03905614320585842, 'num_input_tokens_seen': 5335154688, 'completed': '69.00% (2_544 / 3_687)', 'remaining time': '9:44:18', 'throughput': '8431.91', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 16:08:20,914 >> {'loss': 0.4095, 'grad_norm': 16.298038482666016, 'learning_rate': 3.1620221914808115e-07, 'epoch': 0.03932736642256577, 'num_input_tokens_seen': 5337251840, 'completed': '69.03% (2_545 / 3_687)', 'remaining time': '9:44:00', 'throughput': '8095.61', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 16:08:52,820 >> {'loss': 0.3647, 'grad_norm': 18.965286254882812, 'learning_rate': 3.1585738590718157e-07, 'epoch': 0.039598589639273124, 'num_input_tokens_seen': 5339348992, 'completed': '69.05% (2_546 / 3_687)', 'remaining time': '9:43:39', 'throughput': '8216.21', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 16:09:21,858 >> {'loss': 0.6584, 'grad_norm': 22.938982009887695, 'learning_rate': 3.1551274109534805e-07, 'epoch': 0.03986981285598047, 'num_input_tokens_seen': 5341446144, 'completed': '69.08% (2_547 / 3_687)', 'remaining time': '9:42:56', 'throughput': '9027.63', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 16:09:52,493 >> {'loss': 0.6629, 'grad_norm': 20.005109786987305, 'learning_rate': 3.151682849899376e-07, 'epoch': 0.04014103607268782, 'num_input_tokens_seen': 5343543296, 'completed': '69.11% (2_548 / 3_687)', 'remaining time': '9:42:25', 'throughput': '8556.99', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 16:10:23,366 >> {'loss': 0.4334, 'grad_norm': 16.92494773864746, 'learning_rate': 3.148240178681553e-07, 'epoch': 0.040412259289395173, 'num_input_tokens_seen': 5345640448, 'completed': '69.13% (2_549 / 3_687)', 'remaining time': '9:41:56', 'throughput': '8490.97', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 16:10:56,298 >> {'loss': 0.6027, 'grad_norm': 20.46824073791504, 'learning_rate': 3.1447994000705456e-07, 'epoch': 0.04068348250610252, 'num_input_tokens_seen': 5347737600, 'completed': '69.16% (2_550 / 3_687)', 'remaining time': '9:41:42', 'throughput': '7960.10', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 16:11:25,510 >> {'loss': 0.3774, 'grad_norm': 13.558602333068848, 'learning_rate': 3.141360516835356e-07, 'epoch': 0.040954705722809875, 'num_input_tokens_seen': 5349834752, 'completed': '69.19% (2_551 / 3_687)', 'remaining time': '9:41:00', 'throughput': '8974.01', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 16:11:56,187 >> {'loss': 0.1835, 'grad_norm': 8.050308227539062, 'learning_rate': 3.1379235317434703e-07, 'epoch': 0.04122592893951722, 'num_input_tokens_seen': 5351931904, 'completed': '69.22% (2_552 / 3_687)', 'remaining time': '9:40:29', 'throughput': '8545.32', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 16:12:26,801 >> {'loss': 0.3894, 'grad_norm': 15.542895317077637, 'learning_rate': 3.134488447560843e-07, 'epoch': 0.04149715215622457, 'num_input_tokens_seen': 5354029056, 'completed': '69.24% (2_553 / 3_687)', 'remaining time': '9:39:58', 'throughput': '8562.72', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 16:12:57,223 >> {'loss': 0.7527, 'grad_norm': 25.710311889648438, 'learning_rate': 3.1310552670518987e-07, 'epoch': 0.041768375372931925, 'num_input_tokens_seen': 5356126208, 'completed': '69.27% (2_554 / 3_687)', 'remaining time': '9:39:25', 'throughput': '8616.97', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 16:13:26,525 >> {'loss': 0.4483, 'grad_norm': 19.23603057861328, 'learning_rate': 3.127623992979532e-07, 'epoch': 0.04203959858963927, 'num_input_tokens_seen': 5358223360, 'completed': '69.30% (2_555 / 3_687)', 'remaining time': '9:38:45', 'throughput': '8946.33', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 16:13:56,487 >> {'loss': 0.3604, 'grad_norm': 16.744998931884766, 'learning_rate': 3.124194628105098e-07, 'epoch': 0.04231082180634662, 'num_input_tokens_seen': 5360320512, 'completed': '69.32% (2_556 / 3_687)', 'remaining time': '9:38:09', 'throughput': '8749.22', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 16:14:27,419 >> {'loss': 0.5432, 'grad_norm': 19.892236709594727, 'learning_rate': 3.120767175188422e-07, 'epoch': 0.042582045023053974, 'num_input_tokens_seen': 5362417664, 'completed': '69.35% (2_557 / 3_687)', 'remaining time': '9:37:40', 'throughput': '8474.90', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 16:14:55,957 >> {'loss': 0.6695, 'grad_norm': 21.464439392089844, 'learning_rate': 3.1173416369877864e-07, 'epoch': 0.04285326823976132, 'num_input_tokens_seen': 5364514816, 'completed': '69.38% (2_558 / 3_687)', 'remaining time': '9:36:54', 'throughput': '9185.68', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 16:15:25,632 >> {'loss': 0.4755, 'grad_norm': 18.040634155273438, 'learning_rate': 3.1139180162599346e-07, 'epoch': 0.043124491456468676, 'num_input_tokens_seen': 5366611968, 'completed': '69.41% (2_559 / 3_687)', 'remaining time': '9:36:16', 'throughput': '8834.00', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 16:15:57,652 >> {'loss': 0.3587, 'grad_norm': 16.184511184692383, 'learning_rate': 3.110496315760065e-07, 'epoch': 0.043395714673176024, 'num_input_tokens_seen': 5368709120, 'completed': '69.43% (2_560 / 3_687)', 'remaining time': '9:35:55', 'throughput': '8186.87', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 16:16:29,577 >> {'loss': 0.4959, 'grad_norm': 14.748477935791016, 'learning_rate': 3.107076538241835e-07, 'epoch': 0.04366693788988337, 'num_input_tokens_seen': 5370806272, 'completed': '69.46% (2_561 / 3_687)', 'remaining time': '9:35:34', 'throughput': '8211.19', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 16:16:59,308 >> {'loss': 0.3855, 'grad_norm': 15.208799362182617, 'learning_rate': 3.103658686457349e-07, 'epoch': 0.043938161106590726, 'num_input_tokens_seen': 5372903424, 'completed': '69.49% (2_562 / 3_687)', 'remaining time': '9:34:56', 'throughput': '8817.25', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 16:17:28,641 >> {'loss': 0.3119, 'grad_norm': 14.134151458740234, 'learning_rate': 3.1002427631571646e-07, 'epoch': 0.04420938432329807, 'num_input_tokens_seen': 5375000576, 'completed': '69.51% (2_563 / 3_687)', 'remaining time': '9:34:17', 'throughput': '8936.78', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 16:18:02,899 >> {'loss': 0.3008, 'grad_norm': 14.675880432128906, 'learning_rate': 3.0968287710902866e-07, 'epoch': 0.04448060754000543, 'num_input_tokens_seen': 5377097728, 'completed': '69.54% (2_564 / 3_687)', 'remaining time': '9:34:11', 'throughput': '7652.13', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 16:18:29,658 >> {'loss': 0.638, 'grad_norm': 18.92854881286621, 'learning_rate': 3.0934167130041666e-07, 'epoch': 0.044751830756712775, 'num_input_tokens_seen': 5379194880, 'completed': '69.57% (2_565 / 3_687)', 'remaining time': '9:33:13', 'throughput': '9796.45', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 16:18:58,294 >> {'loss': 0.5573, 'grad_norm': 17.33856773376465, 'learning_rate': 3.090006591644698e-07, 'epoch': 0.04502305397342012, 'num_input_tokens_seen': 5381292032, 'completed': '69.60% (2_566 / 3_687)', 'remaining time': '9:32:29', 'throughput': '9154.22', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 16:19:31,905 >> {'loss': 0.2354, 'grad_norm': 10.942505836486816, 'learning_rate': 3.0865984097562183e-07, 'epoch': 0.04529427719012748, 'num_input_tokens_seen': 5383389184, 'completed': '69.62% (2_567 / 3_687)', 'remaining time': '9:32:18', 'throughput': '7799.37', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 16:20:04,401 >> {'loss': 0.5418, 'grad_norm': 17.489700317382812, 'learning_rate': 3.083192170081501e-07, 'epoch': 0.045565500406834825, 'num_input_tokens_seen': 5385486336, 'completed': '69.65% (2_568 / 3_687)', 'remaining time': '9:32:00', 'throughput': '8066.84', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 16:20:34,006 >> {'loss': 0.4361, 'grad_norm': 17.590167999267578, 'learning_rate': 3.079787875361759e-07, 'epoch': 0.04583672362354217, 'num_input_tokens_seen': 5387583488, 'completed': '69.68% (2_569 / 3_687)', 'remaining time': '9:31:22', 'throughput': '8854.78', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 16:21:04,350 >> {'loss': 0.4253, 'grad_norm': 20.286165237426758, 'learning_rate': 3.0763855283366386e-07, 'epoch': 0.04610794684024953, 'num_input_tokens_seen': 5389680640, 'completed': '69.70% (2_570 / 3_687)', 'remaining time': '9:30:49', 'throughput': '8639.15', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 16:21:34,729 >> {'loss': 0.2986, 'grad_norm': 13.312159538269043, 'learning_rate': 3.07298513174422e-07, 'epoch': 0.046379170056956874, 'num_input_tokens_seen': 5391777792, 'completed': '69.73% (2_571 / 3_687)', 'remaining time': '9:30:17', 'throughput': '8629.23', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 16:22:02,937 >> {'loss': 0.5234, 'grad_norm': 17.008472442626953, 'learning_rate': 3.0695866883210143e-07, 'epoch': 0.04665039327366423, 'num_input_tokens_seen': 5393874944, 'completed': '69.76% (2_572 / 3_687)', 'remaining time': '9:29:30', 'throughput': '9292.94', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 16:22:30,369 >> {'loss': 0.6498, 'grad_norm': 25.539731979370117, 'learning_rate': 3.0661902008019556e-07, 'epoch': 0.046921616490371576, 'num_input_tokens_seen': 5395972096, 'completed': '69.79% (2_573 / 3_687)', 'remaining time': '9:28:39', 'throughput': '9556.44', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 16:23:01,782 >> {'loss': 0.8046, 'grad_norm': 23.586692810058594, 'learning_rate': 3.062795671920411e-07, 'epoch': 0.047192839707078924, 'num_input_tokens_seen': 5398069248, 'completed': '69.81% (2_574 / 3_687)', 'remaining time': '9:28:13', 'throughput': '8345.03', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 16:23:34,750 >> {'loss': 0.3061, 'grad_norm': 13.434320449829102, 'learning_rate': 3.059403104408166e-07, 'epoch': 0.04746406292378628, 'num_input_tokens_seen': 5400166400, 'completed': '69.84% (2_575 / 3_687)', 'remaining time': '9:27:57', 'throughput': '7951.49', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 16:24:05,762 >> {'loss': 0.2875, 'grad_norm': 11.639397621154785, 'learning_rate': 3.0560125009954296e-07, 'epoch': 0.047735286140493625, 'num_input_tokens_seen': 5402263552, 'completed': '69.87% (2_576 / 3_687)', 'remaining time': '9:27:29', 'throughput': '8452.79', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 16:24:37,984 >> {'loss': 0.5249, 'grad_norm': 23.569503784179688, 'learning_rate': 3.0526238644108335e-07, 'epoch': 0.04800650935720097, 'num_input_tokens_seen': 5404360704, 'completed': '69.89% (2_577 / 3_687)', 'remaining time': '9:27:08', 'throughput': '8135.75', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 16:25:09,862 >> {'loss': 0.5019, 'grad_norm': 17.796600341796875, 'learning_rate': 3.049237197381424e-07, 'epoch': 0.04827773257390833, 'num_input_tokens_seen': 5406457856, 'completed': '69.92% (2_578 / 3_687)', 'remaining time': '9:26:45', 'throughput': '8223.15', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 16:25:43,848 >> {'loss': 0.5809, 'grad_norm': 18.38736915588379, 'learning_rate': 3.045852502632657e-07, 'epoch': 0.048548955790615675, 'num_input_tokens_seen': 5408555008, 'completed': '69.95% (2_579 / 3_687)', 'remaining time': '9:26:35', 'throughput': '7713.28', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 16:26:15,140 >> {'loss': 0.2202, 'grad_norm': 10.763379096984863, 'learning_rate': 3.042469782888409e-07, 'epoch': 0.04882017900732303, 'num_input_tokens_seen': 5410652160, 'completed': '69.98% (2_580 / 3_687)', 'remaining time': '9:26:08', 'throughput': '8377.42', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 16:26:44,192 >> {'loss': 0.4292, 'grad_norm': 31.74101448059082, 'learning_rate': 3.0390890408709645e-07, 'epoch': 0.04909140222403038, 'num_input_tokens_seen': 5412749312, 'completed': '70.00% (2_581 / 3_687)', 'remaining time': '9:25:28', 'throughput': '9023.21', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 16:27:15,432 >> {'loss': 0.3474, 'grad_norm': 14.116605758666992, 'learning_rate': 3.0357102793010145e-07, 'epoch': 0.049362625440737724, 'num_input_tokens_seen': 5414846464, 'completed': '70.03% (2_582 / 3_687)', 'remaining time': '9:25:00', 'throughput': '8391.53', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 16:27:46,553 >> {'loss': 0.1974, 'grad_norm': 11.441993713378906, 'learning_rate': 3.032333500897659e-07, 'epoch': 0.04963384865744508, 'num_input_tokens_seen': 5416943616, 'completed': '70.06% (2_583 / 3_687)', 'remaining time': '9:24:32', 'throughput': '8423.28', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 16:28:17,946 >> {'loss': 0.4156, 'grad_norm': 17.127378463745117, 'learning_rate': 3.0289587083784e-07, 'epoch': 0.049905071874152426, 'num_input_tokens_seen': 5419040768, 'completed': '70.08% (2_584 / 3_687)', 'remaining time': '9:24:06', 'throughput': '8350.47', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 16:28:51,235 >> {'loss': 0.3239, 'grad_norm': 13.827837944030762, 'learning_rate': 3.0255859044591425e-07, 'epoch': 0.05017629509085978, 'num_input_tokens_seen': 5421137920, 'completed': '70.11% (2_585 / 3_687)', 'remaining time': '9:23:51', 'throughput': '7874.77', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 16:29:21,316 >> {'loss': 0.2857, 'grad_norm': 11.431439399719238, 'learning_rate': 3.02221509185419e-07, 'epoch': 0.05044751830756713, 'num_input_tokens_seen': 5423235072, 'completed': '70.14% (2_586 / 3_687)', 'remaining time': '9:23:16', 'throughput': '8714.61', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 16:29:53,827 >> {'loss': 0.3933, 'grad_norm': 23.875659942626953, 'learning_rate': 3.0188462732762457e-07, 'epoch': 0.050718741524274476, 'num_input_tokens_seen': 5425332224, 'completed': '70.17% (2_587 / 3_687)', 'remaining time': '9:22:56', 'throughput': '8063.11', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 16:30:24,623 >> {'loss': 0.3873, 'grad_norm': 16.398712158203125, 'learning_rate': 3.015479451436406e-07, 'epoch': 0.05098996474098183, 'num_input_tokens_seen': 5427429376, 'completed': '70.19% (2_588 / 3_687)', 'remaining time': '9:22:26', 'throughput': '8512.47', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 16:30:56,282 >> {'loss': 0.4602, 'grad_norm': 17.132213592529297, 'learning_rate': 3.0121146290441593e-07, 'epoch': 0.05126118795768918, 'num_input_tokens_seen': 5429526528, 'completed': '70.22% (2_589 / 3_687)', 'remaining time': '9:22:01', 'throughput': '8280.25', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 16:31:26,993 >> {'loss': 0.6185, 'grad_norm': 21.008514404296875, 'learning_rate': 3.0087518088073905e-07, 'epoch': 0.051532411174396525, 'num_input_tokens_seen': 5431623680, 'completed': '70.25% (2_590 / 3_687)', 'remaining time': '9:21:30', 'throughput': '8535.84', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 16:31:55,220 >> {'loss': 0.4214, 'grad_norm': 19.426393508911133, 'learning_rate': 3.005390993432366e-07, 'epoch': 0.05180363439110388, 'num_input_tokens_seen': 5433720832, 'completed': '70.27% (2_591 / 3_687)', 'remaining time': '9:20:45', 'throughput': '9286.88', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 16:32:27,734 >> {'loss': 0.3765, 'grad_norm': 15.54585075378418, 'learning_rate': 3.00203218562374e-07, 'epoch': 0.05207485760781123, 'num_input_tokens_seen': 5435817984, 'completed': '70.30% (2_592 / 3_687)', 'remaining time': '9:20:25', 'throughput': '8062.38', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 16:33:01,412 >> {'loss': 0.2658, 'grad_norm': 15.346580505371094, 'learning_rate': 2.9986753880845596e-07, 'epoch': 0.05234608082451858, 'num_input_tokens_seen': 5437915136, 'completed': '70.33% (2_593 / 3_687)', 'remaining time': '9:20:11', 'throughput': '7783.79', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 16:33:31,572 >> {'loss': 0.2507, 'grad_norm': 11.858630180358887, 'learning_rate': 2.9953206035162433e-07, 'epoch': 0.05261730404122593, 'num_input_tokens_seen': 5440012288, 'completed': '70.36% (2_594 / 3_687)', 'remaining time': '9:19:37', 'throughput': '8692.01', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 16:34:01,306 >> {'loss': 0.3481, 'grad_norm': 14.775229454040527, 'learning_rate': 2.991967834618597e-07, 'epoch': 0.05288852725793328, 'num_input_tokens_seen': 5442109440, 'completed': '70.38% (2_595 / 3_687)', 'remaining time': '9:19:01', 'throughput': '8816.22', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 16:34:32,526 >> {'loss': 0.3183, 'grad_norm': 12.877067565917969, 'learning_rate': 2.9886170840897977e-07, 'epoch': 0.05315975047464063, 'num_input_tokens_seen': 5444206592, 'completed': '70.41% (2_596 / 3_687)', 'remaining time': '9:18:33', 'throughput': '8396.62', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 16:35:02,082 >> {'loss': 0.4371, 'grad_norm': 20.334400177001953, 'learning_rate': 2.9852683546264047e-07, 'epoch': 0.05343097369134798, 'num_input_tokens_seen': 5446303744, 'completed': '70.44% (2_597 / 3_687)', 'remaining time': '9:17:56', 'throughput': '8869.38', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 16:35:29,931 >> {'loss': 0.7361, 'grad_norm': 24.98883819580078, 'learning_rate': 2.9819216489233467e-07, 'epoch': 0.05370219690805533, 'num_input_tokens_seen': 5448400896, 'completed': '70.46% (2_598 / 3_687)', 'remaining time': '9:17:09', 'throughput': '9413.07', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 16:36:01,651 >> {'loss': 0.3259, 'grad_norm': 12.599200248718262, 'learning_rate': 2.978576969673926e-07, 'epoch': 0.05397342012476268, 'num_input_tokens_seen': 5450498048, 'completed': '70.49% (2_599 / 3_687)', 'remaining time': '9:16:44', 'throughput': '8264.43', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 16:36:31,987 >> {'loss': 0.463, 'grad_norm': 17.292268753051758, 'learning_rate': 2.9752343195698125e-07, 'epoch': 0.05424464334147003, 'num_input_tokens_seen': 5452595200, 'completed': '70.52% (2_600 / 3_687)', 'remaining time': '9:16:12', 'throughput': '8641.26', 'gpu_mem_free': '5581MB'} +/scratch3/workspace/ctpham_umass_edu-ft/envs/prolong-final/lib/python3.10/site-packages/torch/distributed/fsdp/fully_sharded_data_parallel.py:689: FutureWarning: FSDP.state_dict_type() and FSDP.set_state_dict_type() are being deprecated. Please use APIs, get_state_dict() and set_state_dict(), which can support different parallelisms, FSDP1, FSDP2, DDP. API doc: https://pytorch.org/docs/stable/distributed.checkpoint.html#torch.distributed.checkpoint.state_dict.get_state_dict .Tutorial: https://pytorch.org/tutorials/recipes/distributed_checkpoint_recipe.html . + warnings.warn( +[INFO|trainer.py:3503] 2024-12-26 16:36:54,749 >> Saving model checkpoint to /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/checkpoint-2600 +[INFO|configuration_utils.py:472] 2024-12-26 16:36:54,752 >> Configuration saved in /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/checkpoint-2600/config.json +[INFO|configuration_utils.py:807] 2024-12-26 16:36:54,754 >> Configuration saved in /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/checkpoint-2600/generation_config.json +[INFO|modeling_utils.py:2807] 2024-12-26 16:37:51,845 >> The model is bigger than the maximum size per checkpoint (5GB) and is going to be split in 7 checkpoint shards. You can find where each parameters has been saved in the index located at /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/checkpoint-2600/model.safetensors.index.json. +[INFO|tokenization_utils_base.py:2684] 2024-12-26 16:37:51,849 >> tokenizer config file saved in /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/checkpoint-2600/tokenizer_config.json +[INFO|tokenization_utils_base.py:2693] 2024-12-26 16:37:51,850 >> Special tokens file saved in /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/checkpoint-2600/special_tokens_map.json +/scratch3/workspace/ctpham_umass_edu-ft/envs/prolong-final/lib/python3.10/site-packages/torch/distributed/fsdp/fully_sharded_data_parallel.py:689: FutureWarning: FSDP.state_dict_type() and FSDP.set_state_dict_type() are being deprecated. Please use APIs, get_state_dict() and set_state_dict(), which can support different parallelisms, FSDP1, FSDP2, DDP. API doc: https://pytorch.org/docs/stable/distributed.checkpoint.html#torch.distributed.checkpoint.state_dict.get_state_dict .Tutorial: https://pytorch.org/tutorials/recipes/distributed_checkpoint_recipe.html . + warnings.warn( +[WARNING|trainer.py:868] 2024-12-26 16:41:29,149 >> Save streaming dataset state: {'epoch': 0, 'sample_in_epoch': 5200, 'num_canonical_nodes': 1, 'shuffle_seed': 42, 'initial_physical_nodes': 1} +12/26/2024 16:41:29 - WARNING - streaming.base.dataset - Because `shuffle_block_size` was not specified, it will default to max(4_000_000 // num_canonical_nodes, 1 << 18) if num_canonical_nodes is not None, otherwise 262144. Prior to Streaming v0.7.0, `shuffle_block_size` defaulted to 262144. +/scratch3/workspace/ctpham_umass_edu-ft/envs/prolong-final/lib/python3.10/site-packages/torch/utils/checkpoint.py:1399: FutureWarning: `torch.cpu.amp.autocast(args...)` is deprecated. Please use `torch.amp.autocast('cpu', args...)` instead. + with device_autocast_ctx, torch.cpu.amp.autocast(**cpu_autocast_kwargs), recompute_context: # type: ignore[attr-defined] +[INFO|trainer.py:175] 2024-12-26 16:42:00,038 >> {'loss': 0.3436, 'grad_norm': 18.126787185668945, 'learning_rate': 2.9718937013010444e-07, 'epoch': 0.05451586655817738, 'num_input_tokens_seen': 5454692352, 'completed': '70.55% (2_601 / 3_687)', 'remaining time': '9:42:28', 'throughput': '799.09', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 16:42:29,817 >> {'loss': 0.192, 'grad_norm': 9.295369148254395, 'learning_rate': 2.9685551175560235e-07, 'epoch': 0.05478708977488473, 'num_input_tokens_seen': 5456789504, 'completed': '70.57% (2_602 / 3_687)', 'remaining time': '9:41:42', 'throughput': '8803.07', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 16:42:56,578 >> {'loss': 0.7479, 'grad_norm': 21.22579574584961, 'learning_rate': 2.965218571021516e-07, 'epoch': 0.05505831299159208, 'num_input_tokens_seen': 5458886656, 'completed': '70.60% (2_603 / 3_687)', 'remaining time': '9:40:41', 'throughput': '9795.63', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 16:43:25,695 >> {'loss': 0.4214, 'grad_norm': 18.786785125732422, 'learning_rate': 2.9618840643826464e-07, 'epoch': 0.05532953620829943, 'num_input_tokens_seen': 5460983808, 'completed': '70.63% (2_604 / 3_687)', 'remaining time': '9:39:53', 'throughput': '9003.27', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 16:43:58,467 >> {'loss': 0.5877, 'grad_norm': 20.445255279541016, 'learning_rate': 2.958551600322899e-07, 'epoch': 0.05560075942500678, 'num_input_tokens_seen': 5463080960, 'completed': '70.65% (2_605 / 3_687)', 'remaining time': '9:39:24', 'throughput': '7998.86', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 16:44:26,754 >> {'loss': 0.5026, 'grad_norm': 19.826480865478516, 'learning_rate': 2.9552211815241156e-07, 'epoch': 0.055871982641714134, 'num_input_tokens_seen': 5465178112, 'completed': '70.68% (2_606 / 3_687)', 'remaining time': '9:38:32', 'throughput': '9267.58', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 16:45:00,850 >> {'loss': 0.35, 'grad_norm': 14.695202827453613, 'learning_rate': 2.9518928106664897e-07, 'epoch': 0.05614320585842148, 'num_input_tokens_seen': 5467275264, 'completed': '70.71% (2_607 / 3_687)', 'remaining time': '9:38:10', 'throughput': '7688.43', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 16:45:29,284 >> {'loss': 0.4947, 'grad_norm': 16.892658233642578, 'learning_rate': 2.9485664904285643e-07, 'epoch': 0.05641442907512883, 'num_input_tokens_seen': 5469372416, 'completed': '70.74% (2_608 / 3_687)', 'remaining time': '9:37:19', 'throughput': '9219.29', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 16:45:59,155 >> {'loss': 0.5193, 'grad_norm': 19.319557189941406, 'learning_rate': 2.94524222348724e-07, 'epoch': 0.05668565229183618, 'num_input_tokens_seen': 5471469568, 'completed': '70.76% (2_609 / 3_687)', 'remaining time': '9:36:36', 'throughput': '8775.84', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 16:46:30,749 >> {'loss': 0.3985, 'grad_norm': 21.24896240234375, 'learning_rate': 2.9419200125177585e-07, 'epoch': 0.05695687550854353, 'num_input_tokens_seen': 5473566720, 'completed': '70.79% (2_610 / 3_687)', 'remaining time': '9:36:01', 'throughput': '8297.36', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 16:47:03,694 >> {'loss': 0.2865, 'grad_norm': 11.423199653625488, 'learning_rate': 2.938599860193709e-07, 'epoch': 0.05722809872525088, 'num_input_tokens_seen': 5475663872, 'completed': '70.82% (2_611 / 3_687)', 'remaining time': '9:35:33', 'throughput': '7956.94', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 16:47:34,914 >> {'loss': 0.2112, 'grad_norm': 11.481456756591797, 'learning_rate': 2.935281769187025e-07, 'epoch': 0.05749932194195823, 'num_input_tokens_seen': 5477761024, 'completed': '70.84% (2_612 / 3_687)', 'remaining time': '9:34:57', 'throughput': '8396.73', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 16:48:06,460 >> {'loss': 0.4367, 'grad_norm': 19.210725784301758, 'learning_rate': 2.9319657421679757e-07, 'epoch': 0.05777054515866558, 'num_input_tokens_seen': 5479858176, 'completed': '70.87% (2_613 / 3_687)', 'remaining time': '9:34:22', 'throughput': '8309.84', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 16:48:35,589 >> {'loss': 0.3199, 'grad_norm': 17.227354049682617, 'learning_rate': 2.928651781805177e-07, 'epoch': 0.058041768375372935, 'num_input_tokens_seen': 5481955328, 'completed': '70.90% (2_614 / 3_687)', 'remaining time': '9:33:35', 'throughput': '8999.48', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 16:49:05,415 >> {'loss': 0.2538, 'grad_norm': 11.751642227172852, 'learning_rate': 2.9253398907655775e-07, 'epoch': 0.05831299159208028, 'num_input_tokens_seen': 5484052480, 'completed': '70.92% (2_615 / 3_687)', 'remaining time': '9:32:52', 'throughput': '8789.04', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 16:49:34,465 >> {'loss': 0.2633, 'grad_norm': 11.568734169006348, 'learning_rate': 2.9220300717144597e-07, 'epoch': 0.05858421480878763, 'num_input_tokens_seen': 5486149632, 'completed': '70.95% (2_616 / 3_687)', 'remaining time': '9:32:05', 'throughput': '9023.87', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 16:50:05,200 >> {'loss': 0.2719, 'grad_norm': 12.480401992797852, 'learning_rate': 2.9187223273154406e-07, 'epoch': 0.058855438025494984, 'num_input_tokens_seen': 5488246784, 'completed': '70.98% (2_617 / 3_687)', 'remaining time': '9:31:26', 'throughput': '8529.15', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 16:50:40,364 >> {'loss': 0.5558, 'grad_norm': 21.550291061401367, 'learning_rate': 2.91541666023047e-07, 'epoch': 0.05912666124220233, 'num_input_tokens_seen': 5490343936, 'completed': '71.01% (2_618 / 3_687)', 'remaining time': '9:31:09', 'throughput': '7454.83', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 16:51:13,074 >> {'loss': 0.5337, 'grad_norm': 20.7867374420166, 'learning_rate': 2.9121130731198204e-07, 'epoch': 0.059397884458909686, 'num_input_tokens_seen': 5492441088, 'completed': '71.03% (2_619 / 3_687)', 'remaining time': '9:30:40', 'throughput': '8014.35', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 16:51:45,429 >> {'loss': 0.4042, 'grad_norm': 15.338828086853027, 'learning_rate': 2.9088115686420917e-07, 'epoch': 0.059669107675617034, 'num_input_tokens_seen': 5494538240, 'completed': '71.06% (2_620 / 3_687)', 'remaining time': '9:30:10', 'throughput': '8102.04', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 16:52:14,578 >> {'loss': 0.3717, 'grad_norm': 17.88315200805664, 'learning_rate': 2.9055121494542115e-07, 'epoch': 0.05994033089232438, 'num_input_tokens_seen': 5496635392, 'completed': '71.09% (2_621 / 3_687)', 'remaining time': '9:29:24', 'throughput': '8993.32', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 16:52:47,663 >> {'loss': 0.2657, 'grad_norm': 12.472522735595703, 'learning_rate': 2.9022148182114247e-07, 'epoch': 0.060211554109031735, 'num_input_tokens_seen': 5498732544, 'completed': '71.11% (2_622 / 3_687)', 'remaining time': '9:28:57', 'throughput': '7923.40', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 16:53:18,641 >> {'loss': 0.5611, 'grad_norm': 23.858972549438477, 'learning_rate': 2.8989195775673e-07, 'epoch': 0.06048277732573908, 'num_input_tokens_seen': 5500829696, 'completed': '71.14% (2_623 / 3_687)', 'remaining time': '9:28:19', 'throughput': '8462.14', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 16:53:50,116 >> {'loss': 0.3324, 'grad_norm': 12.803933143615723, 'learning_rate': 2.89562643017372e-07, 'epoch': 0.06075400054244643, 'num_input_tokens_seen': 5502926848, 'completed': '71.17% (2_624 / 3_687)', 'remaining time': '9:27:45', 'throughput': '8328.63', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 16:54:21,986 >> {'loss': 0.4797, 'grad_norm': 26.708951950073242, 'learning_rate': 2.8923353786808857e-07, 'epoch': 0.061025223759153785, 'num_input_tokens_seen': 5505024000, 'completed': '71.20% (2_625 / 3_687)', 'remaining time': '9:27:12', 'throughput': '8225.36', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 16:54:50,022 >> {'loss': 0.8212, 'grad_norm': 25.36354637145996, 'learning_rate': 2.8890464257373105e-07, 'epoch': 0.06129644697586113, 'num_input_tokens_seen': 5507121152, 'completed': '71.22% (2_626 / 3_687)', 'remaining time': '9:26:21', 'throughput': '9350.29', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 16:55:20,167 >> {'loss': 0.3743, 'grad_norm': 13.080056190490723, 'learning_rate': 2.8857595739898164e-07, 'epoch': 0.06156767019256849, 'num_input_tokens_seen': 5509218304, 'completed': '71.25% (2_627 / 3_687)', 'remaining time': '9:25:40', 'throughput': '8696.28', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 16:55:49,537 >> {'loss': 0.652, 'grad_norm': 19.789167404174805, 'learning_rate': 2.8824748260835386e-07, 'epoch': 0.061838893409275834, 'num_input_tokens_seen': 5511315456, 'completed': '71.28% (2_628 / 3_687)', 'remaining time': '9:24:56', 'throughput': '8925.38', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 16:56:17,742 >> {'loss': 0.6427, 'grad_norm': 28.163496017456055, 'learning_rate': 2.879192184661918e-07, 'epoch': 0.06211011662598318, 'num_input_tokens_seen': 5513412608, 'completed': '71.30% (2_629 / 3_687)', 'remaining time': '9:24:06', 'throughput': '9294.18', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 16:56:51,166 >> {'loss': 0.353, 'grad_norm': 15.175286293029785, 'learning_rate': 2.8759116523666973e-07, 'epoch': 0.062381339842690536, 'num_input_tokens_seen': 5515509760, 'completed': '71.33% (2_630 / 3_687)', 'remaining time': '9:23:41', 'throughput': '7842.98', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 16:57:24,083 >> {'loss': 0.5979, 'grad_norm': 18.326745986938477, 'learning_rate': 2.872633231837923e-07, 'epoch': 0.06265256305939788, 'num_input_tokens_seen': 5517606912, 'completed': '71.36% (2_631 / 3_687)', 'remaining time': '9:23:13', 'throughput': '7963.85', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 16:57:53,263 >> {'loss': 0.2892, 'grad_norm': 14.984691619873047, 'learning_rate': 2.869356925713946e-07, 'epoch': 0.06292378627610523, 'num_input_tokens_seen': 5519704064, 'completed': '71.39% (2_632 / 3_687)', 'remaining time': '9:22:28', 'throughput': '8983.76', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 16:58:21,394 >> {'loss': 0.6904, 'grad_norm': 20.763242721557617, 'learning_rate': 2.86608273663141e-07, 'epoch': 0.06319500949281258, 'num_input_tokens_seen': 5521801216, 'completed': '71.41% (2_633 / 3_687)', 'remaining time': '9:21:39', 'throughput': '9318.82', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 16:58:49,597 >> {'loss': 0.7099, 'grad_norm': 19.28316307067871, 'learning_rate': 2.8628106672252614e-07, 'epoch': 0.06346623270951994, 'num_input_tokens_seen': 5523898368, 'completed': '71.44% (2_634 / 3_687)', 'remaining time': '9:20:50', 'throughput': '9294.68', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 16:59:23,034 >> {'loss': 0.3436, 'grad_norm': 14.820695877075195, 'learning_rate': 2.859540720128737e-07, 'epoch': 0.06373745592622729, 'num_input_tokens_seen': 5525995520, 'completed': '71.47% (2_635 / 3_687)', 'remaining time': '9:20:24', 'throughput': '7839.99', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 16:59:51,914 >> {'loss': 0.397, 'grad_norm': 17.229780197143555, 'learning_rate': 2.856272897973362e-07, 'epoch': 0.06400867914293464, 'num_input_tokens_seen': 5528092672, 'completed': '71.49% (2_636 / 3_687)', 'remaining time': '9:19:39', 'throughput': '9076.93', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 17:00:21,251 >> {'loss': 0.5819, 'grad_norm': 19.926528930664062, 'learning_rate': 2.853007203388958e-07, 'epoch': 0.06427990235964198, 'num_input_tokens_seen': 5530189824, 'completed': '71.52% (2_637 / 3_687)', 'remaining time': '9:18:55', 'throughput': '8935.54', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 17:00:51,079 >> {'loss': 0.3037, 'grad_norm': 12.162901878356934, 'learning_rate': 2.849743639003631e-07, 'epoch': 0.06455112557634933, 'num_input_tokens_seen': 5532286976, 'completed': '71.55% (2_638 / 3_687)', 'remaining time': '9:18:14', 'throughput': '8788.51', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 17:01:23,451 >> {'loss': 0.4711, 'grad_norm': 20.921995162963867, 'learning_rate': 2.846482207443773e-07, 'epoch': 0.06482234879305669, 'num_input_tokens_seen': 5534384128, 'completed': '71.58% (2_639 / 3_687)', 'remaining time': '9:17:44', 'throughput': '8097.88', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 17:01:53,497 >> {'loss': 0.3534, 'grad_norm': 16.384029388427734, 'learning_rate': 2.843222911334061e-07, 'epoch': 0.06509357200976404, 'num_input_tokens_seen': 5536481280, 'completed': '71.60% (2_640 / 3_687)', 'remaining time': '9:17:04', 'throughput': '8724.79', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 17:02:23,231 >> {'loss': 0.6336, 'grad_norm': 22.67833137512207, 'learning_rate': 2.839965753297452e-07, 'epoch': 0.06536479522647139, 'num_input_tokens_seen': 5538578432, 'completed': '71.63% (2_641 / 3_687)', 'remaining time': '9:16:22', 'throughput': '8816.47', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 17:02:53,056 >> {'loss': 0.3167, 'grad_norm': 12.454771995544434, 'learning_rate': 2.8367107359551835e-07, 'epoch': 0.06563601844317873, 'num_input_tokens_seen': 5540675584, 'completed': '71.66% (2_642 / 3_687)', 'remaining time': '9:15:41', 'throughput': '8789.46', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 17:03:20,404 >> {'loss': 0.5943, 'grad_norm': 18.632858276367188, 'learning_rate': 2.8334578619267683e-07, 'epoch': 0.06590724165988608, 'num_input_tokens_seen': 5542772736, 'completed': '71.68% (2_643 / 3_687)', 'remaining time': '9:14:50', 'throughput': '9585.28', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 17:03:50,252 >> {'loss': 0.2241, 'grad_norm': 10.916572570800781, 'learning_rate': 2.830207133829997e-07, 'epoch': 0.06617846487659344, 'num_input_tokens_seen': 5544869888, 'completed': '71.71% (2_644 / 3_687)', 'remaining time': '9:14:09', 'throughput': '8782.64', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 17:04:21,666 >> {'loss': 0.5303, 'grad_norm': 20.393115997314453, 'learning_rate': 2.8269585542809305e-07, 'epoch': 0.06644968809330079, 'num_input_tokens_seen': 5546967040, 'completed': '71.74% (2_645 / 3_687)', 'remaining time': '9:13:36', 'throughput': '8344.71', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 17:04:53,173 >> {'loss': 0.4077, 'grad_norm': 16.837371826171875, 'learning_rate': 2.8237121258939036e-07, 'epoch': 0.06672091131000814, 'num_input_tokens_seen': 5549064192, 'completed': '71.77% (2_646 / 3_687)', 'remaining time': '9:13:02', 'throughput': '8320.42', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 17:05:23,438 >> {'loss': 0.518, 'grad_norm': 25.16261863708496, 'learning_rate': 2.8204678512815185e-07, 'epoch': 0.06699213452671549, 'num_input_tokens_seen': 5551161344, 'completed': '71.79% (2_647 / 3_687)', 'remaining time': '9:12:23', 'throughput': '8661.38', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 17:05:54,318 >> {'loss': 0.572, 'grad_norm': 21.067689895629883, 'learning_rate': 2.8172257330546414e-07, 'epoch': 0.06726335774342283, 'num_input_tokens_seen': 5553258496, 'completed': '71.82% (2_648 / 3_687)', 'remaining time': '9:11:47', 'throughput': '8489.31', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 17:06:26,262 >> {'loss': 0.3443, 'grad_norm': 16.49931526184082, 'learning_rate': 2.8139857738224055e-07, 'epoch': 0.06753458096013018, 'num_input_tokens_seen': 5555355648, 'completed': '71.85% (2_649 / 3_687)', 'remaining time': '9:11:16', 'throughput': '8206.20', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 17:06:56,983 >> {'loss': 0.3556, 'grad_norm': 15.077832221984863, 'learning_rate': 2.81074797619221e-07, 'epoch': 0.06780580417683754, 'num_input_tokens_seen': 5557452800, 'completed': '71.87% (2_650 / 3_687)', 'remaining time': '9:10:39', 'throughput': '8533.10', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 17:07:28,742 >> {'loss': 0.3608, 'grad_norm': 14.75187873840332, 'learning_rate': 2.8075123427697093e-07, 'epoch': 0.06807702739354489, 'num_input_tokens_seen': 5559549952, 'completed': '71.90% (2_651 / 3_687)', 'remaining time': '9:10:07', 'throughput': '8254.10', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 17:07:58,372 >> {'loss': 0.2241, 'grad_norm': 11.628194808959961, 'learning_rate': 2.8042788761588204e-07, 'epoch': 0.06834825061025224, 'num_input_tokens_seen': 5561647104, 'completed': '71.93% (2_652 / 3_687)', 'remaining time': '9:09:26', 'throughput': '8847.44', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 17:08:27,929 >> {'loss': 0.5842, 'grad_norm': 17.89313316345215, 'learning_rate': 2.8010475789617105e-07, 'epoch': 0.06861947382695958, 'num_input_tokens_seen': 5563744256, 'completed': '71.96% (2_653 / 3_687)', 'remaining time': '9:08:45', 'throughput': '8869.19', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 17:08:59,580 >> {'loss': 0.5279, 'grad_norm': 17.917509078979492, 'learning_rate': 2.797818453778806e-07, 'epoch': 0.06889069704366693, 'num_input_tokens_seen': 5565841408, 'completed': '71.98% (2_654 / 3_687)', 'remaining time': '9:08:12', 'throughput': '8282.30', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 17:09:30,158 >> {'loss': 0.3903, 'grad_norm': 14.405590057373047, 'learning_rate': 2.794591503208785e-07, 'epoch': 0.0691619202603743, 'num_input_tokens_seen': 5567938560, 'completed': '72.01% (2_655 / 3_687)', 'remaining time': '9:07:35', 'throughput': '8572.76', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 17:10:01,728 >> {'loss': 0.6449, 'grad_norm': 22.006885528564453, 'learning_rate': 2.791366729848574e-07, 'epoch': 0.06943314347708164, 'num_input_tokens_seen': 5570035712, 'completed': '72.04% (2_656 / 3_687)', 'remaining time': '9:07:02', 'throughput': '8303.77', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 17:10:33,415 >> {'loss': 0.6725, 'grad_norm': 23.890174865722656, 'learning_rate': 2.7881441362933464e-07, 'epoch': 0.06970436669378899, 'num_input_tokens_seen': 5572132864, 'completed': '72.06% (2_657 / 3_687)', 'remaining time': '9:06:30', 'throughput': '8272.78', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 17:11:07,508 >> {'loss': 0.5287, 'grad_norm': 19.865018844604492, 'learning_rate': 2.784923725136525e-07, 'epoch': 0.06997558991049634, 'num_input_tokens_seen': 5574230016, 'completed': '72.09% (2_658 / 3_687)', 'remaining time': '9:06:07', 'throughput': '7689.20', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 17:11:41,630 >> {'loss': 0.2702, 'grad_norm': 9.786247253417969, 'learning_rate': 2.781705498969773e-07, 'epoch': 0.07024681312720368, 'num_input_tokens_seen': 5576327168, 'completed': '72.12% (2_659 / 3_687)', 'remaining time': '9:05:44', 'throughput': '7682.37', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 17:12:15,097 >> {'loss': 0.3915, 'grad_norm': 13.455928802490234, 'learning_rate': 2.7784894603829966e-07, 'epoch': 0.07051803634391104, 'num_input_tokens_seen': 5578424320, 'completed': '72.15% (2_660 / 3_687)', 'remaining time': '9:05:19', 'throughput': '7833.11', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 17:12:45,687 >> {'loss': 0.4046, 'grad_norm': 15.833970069885254, 'learning_rate': 2.7752756119643416e-07, 'epoch': 0.07078925956061839, 'num_input_tokens_seen': 5580521472, 'completed': '72.17% (2_661 / 3_687)', 'remaining time': '9:04:42', 'throughput': '8569.60', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 17:13:16,213 >> {'loss': 0.233, 'grad_norm': 13.186774253845215, 'learning_rate': 2.77206395630019e-07, 'epoch': 0.07106048277732574, 'num_input_tokens_seen': 5582618624, 'completed': '72.20% (2_662 / 3_687)', 'remaining time': '9:04:05', 'throughput': '8587.43', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 17:13:46,136 >> {'loss': 0.3992, 'grad_norm': 16.846803665161133, 'learning_rate': 2.7688544959751615e-07, 'epoch': 0.07133170599403309, 'num_input_tokens_seen': 5584715776, 'completed': '72.23% (2_663 / 3_687)', 'remaining time': '9:03:25', 'throughput': '8760.67', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 17:14:18,342 >> {'loss': 0.454, 'grad_norm': 16.44211769104004, 'learning_rate': 2.765647233572108e-07, 'epoch': 0.07160292921074043, 'num_input_tokens_seen': 5586812928, 'completed': '72.25% (2_664 / 3_687)', 'remaining time': '9:02:55', 'throughput': '8139.54', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 17:14:45,410 >> {'loss': 0.6375, 'grad_norm': 21.622333526611328, 'learning_rate': 2.7624421716721086e-07, 'epoch': 0.0718741524274478, 'num_input_tokens_seen': 5588910080, 'completed': '72.28% (2_665 / 3_687)', 'remaining time': '9:02:05', 'throughput': '9684.69', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 17:15:14,628 >> {'loss': 0.3815, 'grad_norm': 13.752047538757324, 'learning_rate': 2.7592393128544784e-07, 'epoch': 0.07214537564415514, 'num_input_tokens_seen': 5591007232, 'completed': '72.31% (2_666 / 3_687)', 'remaining time': '9:01:23', 'throughput': '8971.92', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 17:15:44,049 >> {'loss': 0.3299, 'grad_norm': 15.634025573730469, 'learning_rate': 2.7560386596967553e-07, 'epoch': 0.07241659886086249, 'num_input_tokens_seen': 5593104384, 'completed': '72.34% (2_667 / 3_687)', 'remaining time': '9:00:42', 'throughput': '8910.02', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 17:16:14,338 >> {'loss': 0.8418, 'grad_norm': 22.741350173950195, 'learning_rate': 2.7528402147747045e-07, 'epoch': 0.07268782207756984, 'num_input_tokens_seen': 5595201536, 'completed': '72.36% (2_668 / 3_687)', 'remaining time': '9:00:04', 'throughput': '8654.78', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 17:16:41,645 >> {'loss': 0.5166, 'grad_norm': 18.136112213134766, 'learning_rate': 2.7496439806623124e-07, 'epoch': 0.07295904529427719, 'num_input_tokens_seen': 5597298688, 'completed': '72.39% (2_669 / 3_687)', 'remaining time': '8:59:16', 'throughput': '9600.01', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 17:17:12,897 >> {'loss': 0.2879, 'grad_norm': 13.299650192260742, 'learning_rate': 2.746449959931784e-07, 'epoch': 0.07323026851098453, 'num_input_tokens_seen': 5599395840, 'completed': '72.42% (2_670 / 3_687)', 'remaining time': '8:58:42', 'throughput': '8388.13', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 17:17:47,022 >> {'loss': 0.2716, 'grad_norm': 12.364994049072266, 'learning_rate': 2.743258155153548e-07, 'epoch': 0.0735014917276919, 'num_input_tokens_seen': 5601492992, 'completed': '72.44% (2_671 / 3_687)', 'remaining time': '8:58:19', 'throughput': '7681.95', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 17:18:18,421 >> {'loss': 0.6404, 'grad_norm': 22.236961364746094, 'learning_rate': 2.740068568896247e-07, 'epoch': 0.07377271494439924, 'num_input_tokens_seen': 5603590144, 'completed': '72.47% (2_672 / 3_687)', 'remaining time': '8:57:46', 'throughput': '8348.64', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 17:18:49,192 >> {'loss': 0.646, 'grad_norm': 18.271963119506836, 'learning_rate': 2.7368812037267387e-07, 'epoch': 0.07404393816110659, 'num_input_tokens_seen': 5605687296, 'completed': '72.50% (2_673 / 3_687)', 'remaining time': '8:57:10', 'throughput': '8519.30', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 17:19:17,777 >> {'loss': 0.7859, 'grad_norm': 22.82843017578125, 'learning_rate': 2.7336960622100907e-07, 'epoch': 0.07431516137781394, 'num_input_tokens_seen': 5607784448, 'completed': '72.53% (2_674 / 3_687)', 'remaining time': '8:56:26', 'throughput': '9170.45', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 17:19:47,968 >> {'loss': 0.1964, 'grad_norm': 11.289409637451172, 'learning_rate': 2.7305131469095906e-07, 'epoch': 0.07458638459452128, 'num_input_tokens_seen': 5609881600, 'completed': '72.55% (2_675 / 3_687)', 'remaining time': '8:55:49', 'throughput': '8683.03', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 17:20:17,867 >> {'loss': 0.5049, 'grad_norm': 20.22431182861328, 'learning_rate': 2.7273324603867203e-07, 'epoch': 0.07485760781122865, 'num_input_tokens_seen': 5611978752, 'completed': '72.58% (2_676 / 3_687)', 'remaining time': '8:55:10', 'throughput': '8767.54', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 17:20:47,863 >> {'loss': 0.6532, 'grad_norm': 21.272485733032227, 'learning_rate': 2.7241540052011787e-07, 'epoch': 0.075128831027936, 'num_input_tokens_seen': 5614075904, 'completed': '72.61% (2_677 / 3_687)', 'remaining time': '8:54:32', 'throughput': '8739.36', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 17:21:16,900 >> {'loss': 0.2299, 'grad_norm': 10.40255069732666, 'learning_rate': 2.720977783910865e-07, 'epoch': 0.07540005424464334, 'num_input_tokens_seen': 5616173056, 'completed': '72.63% (2_678 / 3_687)', 'remaining time': '8:53:50', 'throughput': '9027.92', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 17:21:46,174 >> {'loss': 0.4719, 'grad_norm': 18.323423385620117, 'learning_rate': 2.717803799071881e-07, 'epoch': 0.07567127746135069, 'num_input_tokens_seen': 5618270208, 'completed': '72.66% (2_679 / 3_687)', 'remaining time': '8:53:10', 'throughput': '8954.76', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 17:22:16,738 >> {'loss': 0.5637, 'grad_norm': 16.696556091308594, 'learning_rate': 2.714632053238529e-07, 'epoch': 0.07594250067805804, 'num_input_tokens_seen': 5620367360, 'completed': '72.69% (2_680 / 3_687)', 'remaining time': '8:52:34', 'throughput': '8577.07', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 17:22:47,589 >> {'loss': 0.3831, 'grad_norm': 17.217605590820312, 'learning_rate': 2.711462548963309e-07, 'epoch': 0.0762137238947654, 'num_input_tokens_seen': 5622464512, 'completed': '72.71% (2_681 / 3_687)', 'remaining time': '8:51:59', 'throughput': '8497.06', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 17:23:16,983 >> {'loss': 0.4221, 'grad_norm': 17.2362003326416, 'learning_rate': 2.708295288796918e-07, 'epoch': 0.07648494711147275, 'num_input_tokens_seen': 5624561664, 'completed': '72.74% (2_682 / 3_687)', 'remaining time': '8:51:19', 'throughput': '8918.24', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 17:23:46,342 >> {'loss': 0.5495, 'grad_norm': 19.473421096801758, 'learning_rate': 2.7051302752882467e-07, 'epoch': 0.07675617032818009, 'num_input_tokens_seen': 5626658816, 'completed': '72.77% (2_683 / 3_687)', 'remaining time': '8:50:39', 'throughput': '8928.96', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 17:24:18,509 >> {'loss': 0.4145, 'grad_norm': 18.977439880371094, 'learning_rate': 2.7019675109843777e-07, 'epoch': 0.07702739354488744, 'num_input_tokens_seen': 5628755968, 'completed': '72.80% (2_684 / 3_687)', 'remaining time': '8:50:09', 'throughput': '8149.52', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 17:24:49,105 >> {'loss': 0.5008, 'grad_norm': 22.364627838134766, 'learning_rate': 2.6988069984305817e-07, 'epoch': 0.07729861676159479, 'num_input_tokens_seen': 5630853120, 'completed': '72.82% (2_685 / 3_687)', 'remaining time': '8:49:33', 'throughput': '8567.95', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 17:25:19,892 >> {'loss': 0.5733, 'grad_norm': 19.7006893157959, 'learning_rate': 2.6956487401703207e-07, 'epoch': 0.07756983997830215, 'num_input_tokens_seen': 5632950272, 'completed': '72.85% (2_686 / 3_687)', 'remaining time': '8:48:58', 'throughput': '8514.69', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 17:25:50,997 >> {'loss': 0.662, 'grad_norm': 22.176889419555664, 'learning_rate': 2.692492738745241e-07, 'epoch': 0.0778410631950095, 'num_input_tokens_seen': 5635047424, 'completed': '72.88% (2_687 / 3_687)', 'remaining time': '8:48:24', 'throughput': '8427.58', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 17:26:17,416 >> {'loss': 0.7364, 'grad_norm': 22.010507583618164, 'learning_rate': 2.689338996695171e-07, 'epoch': 0.07811228641171684, 'num_input_tokens_seen': 5637144576, 'completed': '72.90% (2_688 / 3_687)', 'remaining time': '8:47:34', 'throughput': '9922.55', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 17:26:47,422 >> {'loss': 0.3851, 'grad_norm': 13.696541786193848, 'learning_rate': 2.686187516558122e-07, 'epoch': 0.07838350962842419, 'num_input_tokens_seen': 5639241728, 'completed': '72.93% (2_689 / 3_687)', 'remaining time': '8:46:57', 'throughput': '8736.51', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 17:27:21,578 >> {'loss': 0.571, 'grad_norm': 27.36061668395996, 'learning_rate': 2.683038300870285e-07, 'epoch': 0.07865473284513154, 'num_input_tokens_seen': 5641338880, 'completed': '72.96% (2_690 / 3_687)', 'remaining time': '8:46:33', 'throughput': '7674.77', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 17:27:51,715 >> {'loss': 0.3617, 'grad_norm': 14.405625343322754, 'learning_rate': 2.679891352166032e-07, 'epoch': 0.07892595606183889, 'num_input_tokens_seen': 5643436032, 'completed': '72.99% (2_691 / 3_687)', 'remaining time': '8:45:56', 'throughput': '8698.40', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 17:28:22,486 >> {'loss': 0.552, 'grad_norm': 19.312789916992188, 'learning_rate': 2.6767466729779074e-07, 'epoch': 0.07919717927854625, 'num_input_tokens_seen': 5645533184, 'completed': '73.01% (2_692 / 3_687)', 'remaining time': '8:45:22', 'throughput': '8519.22', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 17:28:54,365 >> {'loss': 0.5278, 'grad_norm': 23.497774124145508, 'learning_rate': 2.6736042658366255e-07, 'epoch': 0.0794684024952536, 'num_input_tokens_seen': 5647630336, 'completed': '73.04% (2_693 / 3_687)', 'remaining time': '8:44:51', 'throughput': '8223.10', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 17:29:25,986 >> {'loss': 0.5603, 'grad_norm': 18.922243118286133, 'learning_rate': 2.6704641332710783e-07, 'epoch': 0.07973962571196094, 'num_input_tokens_seen': 5649727488, 'completed': '73.07% (2_694 / 3_687)', 'remaining time': '8:44:19', 'throughput': '8290.15', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 17:30:01,160 >> {'loss': 0.4909, 'grad_norm': 18.824480056762695, 'learning_rate': 2.6673262778083246e-07, 'epoch': 0.08001084892866829, 'num_input_tokens_seen': 5651824640, 'completed': '73.09% (2_695 / 3_687)', 'remaining time': '8:43:59', 'throughput': '7452.89', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 17:30:29,631 >> {'loss': 0.4259, 'grad_norm': 18.664011001586914, 'learning_rate': 2.6641907019735914e-07, 'epoch': 0.08028207214537564, 'num_input_tokens_seen': 5653921792, 'completed': '73.12% (2_696 / 3_687)', 'remaining time': '8:43:16', 'throughput': '9207.36', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 17:30:59,482 >> {'loss': 0.2075, 'grad_norm': 12.166492462158203, 'learning_rate': 2.6610574082902704e-07, 'epoch': 0.080553295362083, 'num_input_tokens_seen': 5656018944, 'completed': '73.15% (2_697 / 3_687)', 'remaining time': '8:42:39', 'throughput': '8781.70', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 17:31:29,714 >> {'loss': 0.5047, 'grad_norm': 17.41253662109375, 'learning_rate': 2.657926399279918e-07, 'epoch': 0.08082451857879035, 'num_input_tokens_seen': 5658116096, 'completed': '73.18% (2_698 / 3_687)', 'remaining time': '8:42:02', 'throughput': '8671.25', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 17:32:00,251 >> {'loss': 0.2702, 'grad_norm': 12.47386646270752, 'learning_rate': 2.65479767746225e-07, 'epoch': 0.0810957417954977, 'num_input_tokens_seen': 5660213248, 'completed': '73.20% (2_699 / 3_687)', 'remaining time': '8:41:27', 'throughput': '8584.30', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 17:32:32,871 >> {'loss': 0.2429, 'grad_norm': 15.2111234664917, 'learning_rate': 2.651671245355144e-07, 'epoch': 0.08136696501220504, 'num_input_tokens_seen': 5662310400, 'completed': '73.23% (2_700 / 3_687)', 'remaining time': '8:40:58', 'throughput': '8036.42', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 17:33:06,060 >> {'loss': 0.4939, 'grad_norm': 20.175670623779297, 'learning_rate': 2.6485471054746315e-07, 'epoch': 0.08163818822891239, 'num_input_tokens_seen': 5664407552, 'completed': '73.26% (2_701 / 3_687)', 'remaining time': '8:40:31', 'throughput': '7898.41', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 17:33:36,667 >> {'loss': 0.4021, 'grad_norm': 15.249000549316406, 'learning_rate': 2.645425260334904e-07, 'epoch': 0.08190941144561975, 'num_input_tokens_seen': 5666504704, 'completed': '73.28% (2_702 / 3_687)', 'remaining time': '8:39:56', 'throughput': '8565.01', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 17:34:08,792 >> {'loss': 0.7402, 'grad_norm': 23.084077835083008, 'learning_rate': 2.6423057124483015e-07, 'epoch': 0.0821806346623271, 'num_input_tokens_seen': 5668601856, 'completed': '73.31% (2_703 / 3_687)', 'remaining time': '8:39:26', 'throughput': '8159.97', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 17:34:38,839 >> {'loss': 0.2521, 'grad_norm': 11.628067016601562, 'learning_rate': 2.6391884643253197e-07, 'epoch': 0.08245185787903445, 'num_input_tokens_seen': 5670699008, 'completed': '73.34% (2_704 / 3_687)', 'remaining time': '8:38:49', 'throughput': '8724.45', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 17:35:13,385 >> {'loss': 0.3268, 'grad_norm': 14.505975723266602, 'learning_rate': 2.6360735184745984e-07, 'epoch': 0.0827230810957418, 'num_input_tokens_seen': 5672796160, 'completed': '73.37% (2_705 / 3_687)', 'remaining time': '8:38:27', 'throughput': '7588.22', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 17:35:42,883 >> {'loss': 0.4597, 'grad_norm': 16.887388229370117, 'learning_rate': 2.6329608774029285e-07, 'epoch': 0.08299430431244914, 'num_input_tokens_seen': 5674893312, 'completed': '73.39% (2_706 / 3_687)', 'remaining time': '8:37:48', 'throughput': '8886.95', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 17:36:13,210 >> {'loss': 0.6038, 'grad_norm': 24.89675521850586, 'learning_rate': 2.6298505436152457e-07, 'epoch': 0.0832655275291565, 'num_input_tokens_seen': 5676990464, 'completed': '73.42% (2_707 / 3_687)', 'remaining time': '8:37:12', 'throughput': '8643.87', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 17:36:44,602 >> {'loss': 0.3616, 'grad_norm': 16.737106323242188, 'learning_rate': 2.626742519614629e-07, 'epoch': 0.08353675074586385, 'num_input_tokens_seen': 5679087616, 'completed': '73.45% (2_708 / 3_687)', 'remaining time': '8:36:40', 'throughput': '8350.71', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 17:37:15,327 >> {'loss': 0.4107, 'grad_norm': 15.908254623413086, 'learning_rate': 2.623636807902298e-07, 'epoch': 0.0838079739625712, 'num_input_tokens_seen': 5681184768, 'completed': '73.47% (2_709 / 3_687)', 'remaining time': '8:36:05', 'throughput': '8532.08', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 17:37:46,392 >> {'loss': 0.2693, 'grad_norm': 12.272135734558105, 'learning_rate': 2.620533410977609e-07, 'epoch': 0.08407919717927854, 'num_input_tokens_seen': 5683281920, 'completed': '73.50% (2_710 / 3_687)', 'remaining time': '8:35:31', 'throughput': '8438.48', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 17:38:17,307 >> {'loss': 0.5119, 'grad_norm': 18.592920303344727, 'learning_rate': 2.617432331338059e-07, 'epoch': 0.08435042039598589, 'num_input_tokens_seen': 5685379072, 'completed': '73.53% (2_711 / 3_687)', 'remaining time': '8:34:57', 'throughput': '8479.36', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 17:38:46,892 >> {'loss': 0.6332, 'grad_norm': 21.81186294555664, 'learning_rate': 2.614333571479279e-07, 'epoch': 0.08462164361269324, 'num_input_tokens_seen': 5687476224, 'completed': '73.56% (2_712 / 3_687)', 'remaining time': '8:34:19', 'throughput': '8860.75', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 17:39:18,701 >> {'loss': 0.2644, 'grad_norm': 13.03216552734375, 'learning_rate': 2.6112371338950325e-07, 'epoch': 0.0848928668294006, 'num_input_tokens_seen': 5689573376, 'completed': '73.58% (2_713 / 3_687)', 'remaining time': '8:33:48', 'throughput': '8241.34', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 17:39:49,431 >> {'loss': 0.5831, 'grad_norm': 21.6326904296875, 'learning_rate': 2.608143021077215e-07, 'epoch': 0.08516409004610795, 'num_input_tokens_seen': 5691670528, 'completed': '73.61% (2_714 / 3_687)', 'remaining time': '8:33:14', 'throughput': '8530.46', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 17:40:20,825 >> {'loss': 0.3289, 'grad_norm': 13.137306213378906, 'learning_rate': 2.6050512355158503e-07, 'epoch': 0.0854353132628153, 'num_input_tokens_seen': 5693767680, 'completed': '73.64% (2_715 / 3_687)', 'remaining time': '8:32:41', 'throughput': '8350.03', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 17:40:50,522 >> {'loss': 0.687, 'grad_norm': 20.455364227294922, 'learning_rate': 2.601961779699091e-07, 'epoch': 0.08570653647952264, 'num_input_tokens_seen': 5695864832, 'completed': '73.66% (2_716 / 3_687)', 'remaining time': '8:32:04', 'throughput': '8827.48', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 17:41:20,890 >> {'loss': 0.3623, 'grad_norm': 14.189478874206543, 'learning_rate': 2.5988746561132136e-07, 'epoch': 0.08597775969622999, 'num_input_tokens_seen': 5697961984, 'completed': '73.69% (2_717 / 3_687)', 'remaining time': '8:31:28', 'throughput': '8632.26', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 17:41:54,548 >> {'loss': 0.3914, 'grad_norm': 14.868339538574219, 'learning_rate': 2.595789867242616e-07, 'epoch': 0.08624898291293735, 'num_input_tokens_seen': 5700059136, 'completed': '73.72% (2_718 / 3_687)', 'remaining time': '8:31:03', 'throughput': '7788.38', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 17:42:26,035 >> {'loss': 0.3688, 'grad_norm': 16.712444305419922, 'learning_rate': 2.5927074155698203e-07, 'epoch': 0.0865202061296447, 'num_input_tokens_seen': 5702156288, 'completed': '73.75% (2_719 / 3_687)', 'remaining time': '8:30:30', 'throughput': '8325.42', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 17:42:54,459 >> {'loss': 0.7762, 'grad_norm': 23.333152770996094, 'learning_rate': 2.589627303575465e-07, 'epoch': 0.08679142934635205, 'num_input_tokens_seen': 5704253440, 'completed': '73.77% (2_720 / 3_687)', 'remaining time': '8:29:49', 'throughput': '9222.63', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 17:43:23,589 >> {'loss': 0.4798, 'grad_norm': 21.890504837036133, 'learning_rate': 2.5865495337383103e-07, 'epoch': 0.0870626525630594, 'num_input_tokens_seen': 5706350592, 'completed': '73.80% (2_721 / 3_687)', 'remaining time': '8:29:10', 'throughput': '8999.15', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 17:43:55,720 >> {'loss': 0.4479, 'grad_norm': 17.646825790405273, 'learning_rate': 2.5834741085352223e-07, 'epoch': 0.08733387577976674, 'num_input_tokens_seen': 5708447744, 'completed': '73.83% (2_722 / 3_687)', 'remaining time': '8:28:40', 'throughput': '8158.53', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 17:44:21,440 >> {'loss': 0.6757, 'grad_norm': 20.30681800842285, 'learning_rate': 2.5804010304411914e-07, 'epoch': 0.0876050989964741, 'num_input_tokens_seen': 5710544896, 'completed': '73.85% (2_723 / 3_687)', 'remaining time': '8:27:50', 'throughput': '10192.43', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 17:44:50,241 >> {'loss': 0.2059, 'grad_norm': 9.638075828552246, 'learning_rate': 2.5773303019293123e-07, 'epoch': 0.08787632221318145, 'num_input_tokens_seen': 5712642048, 'completed': '73.88% (2_724 / 3_687)', 'remaining time': '8:27:11', 'throughput': '9101.85', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 17:45:20,378 >> {'loss': 0.1925, 'grad_norm': 11.49023723602295, 'learning_rate': 2.5742619254707905e-07, 'epoch': 0.0881475454298888, 'num_input_tokens_seen': 5714739200, 'completed': '73.91% (2_725 / 3_687)', 'remaining time': '8:26:35', 'throughput': '8698.52', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 17:45:49,855 >> {'loss': 0.4855, 'grad_norm': 18.779539108276367, 'learning_rate': 2.5711959035349396e-07, 'epoch': 0.08841876864659615, 'num_input_tokens_seen': 5716836352, 'completed': '73.94% (2_726 / 3_687)', 'remaining time': '8:25:57', 'throughput': '8893.03', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 17:46:21,969 >> {'loss': 0.2486, 'grad_norm': 11.857897758483887, 'learning_rate': 2.568132238589175e-07, 'epoch': 0.0886899918633035, 'num_input_tokens_seen': 5718933504, 'completed': '73.96% (2_727 / 3_687)', 'remaining time': '8:25:27', 'throughput': '8162.90', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 17:46:51,592 >> {'loss': 0.4293, 'grad_norm': 20.438987731933594, 'learning_rate': 2.565070933099019e-07, 'epoch': 0.08896121508001086, 'num_input_tokens_seen': 5721030656, 'completed': '73.99% (2_728 / 3_687)', 'remaining time': '8:24:49', 'throughput': '8849.54', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 17:47:20,518 >> {'loss': 0.3677, 'grad_norm': 15.0324125289917, 'learning_rate': 2.5620119895280935e-07, 'epoch': 0.0892324382967182, 'num_input_tokens_seen': 5723127808, 'completed': '74.02% (2_729 / 3_687)', 'remaining time': '8:24:10', 'throughput': '9062.31', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 17:47:53,774 >> {'loss': 0.3675, 'grad_norm': 15.559118270874023, 'learning_rate': 2.5589554103381195e-07, 'epoch': 0.08950366151342555, 'num_input_tokens_seen': 5725224960, 'completed': '74.04% (2_730 / 3_687)', 'remaining time': '8:23:43', 'throughput': '7882.80', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 17:48:27,155 >> {'loss': 0.425, 'grad_norm': 19.351423263549805, 'learning_rate': 2.5559011979889155e-07, 'epoch': 0.0897748847301329, 'num_input_tokens_seen': 5727322112, 'completed': '74.07% (2_731 / 3_687)', 'remaining time': '8:23:17', 'throughput': '7852.93', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 17:48:56,957 >> {'loss': 0.3441, 'grad_norm': 13.77000904083252, 'learning_rate': 2.552849354938399e-07, 'epoch': 0.09004610794684025, 'num_input_tokens_seen': 5729419264, 'completed': '74.10% (2_732 / 3_687)', 'remaining time': '8:22:40', 'throughput': '8796.31', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 17:49:26,885 >> {'loss': 0.3423, 'grad_norm': 13.648260116577148, 'learning_rate': 2.549799883642573e-07, 'epoch': 0.09031733116354759, 'num_input_tokens_seen': 5731516416, 'completed': '74.13% (2_733 / 3_687)', 'remaining time': '8:22:04', 'throughput': '8759.16', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 17:49:57,965 >> {'loss': 0.2816, 'grad_norm': 12.919384002685547, 'learning_rate': 2.5467527865555366e-07, 'epoch': 0.09058855438025495, 'num_input_tokens_seen': 5733613568, 'completed': '74.15% (2_734 / 3_687)', 'remaining time': '8:21:31', 'throughput': '8434.34', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 17:50:25,769 >> {'loss': 0.6129, 'grad_norm': 20.780710220336914, 'learning_rate': 2.5437080661294785e-07, 'epoch': 0.0908597775969623, 'num_input_tokens_seen': 5735710720, 'completed': '74.18% (2_735 / 3_687)', 'remaining time': '8:20:49', 'throughput': '9428.36', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 17:50:56,304 >> {'loss': 0.5003, 'grad_norm': 22.01446533203125, 'learning_rate': 2.5406657248146735e-07, 'epoch': 0.09113100081366965, 'num_input_tokens_seen': 5737807872, 'completed': '74.21% (2_736 / 3_687)', 'remaining time': '8:20:14', 'throughput': '8585.18', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 17:51:28,603 >> {'loss': 0.4947, 'grad_norm': 19.538734436035156, 'learning_rate': 2.5376257650594823e-07, 'epoch': 0.091402224030377, 'num_input_tokens_seen': 5739905024, 'completed': '74.23% (2_737 / 3_687)', 'remaining time': '8:19:45', 'throughput': '8115.99', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 17:51:59,693 >> {'loss': 0.2547, 'grad_norm': 13.193866729736328, 'learning_rate': 2.5345881893103484e-07, 'epoch': 0.09167344724708434, 'num_input_tokens_seen': 5742002176, 'completed': '74.26% (2_738 / 3_687)', 'remaining time': '8:19:12', 'throughput': '8431.76', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 17:52:29,821 >> {'loss': 0.7383, 'grad_norm': 23.290098190307617, 'learning_rate': 2.5315530000117973e-07, 'epoch': 0.0919446704637917, 'num_input_tokens_seen': 5744099328, 'completed': '74.29% (2_739 / 3_687)', 'remaining time': '8:18:36', 'throughput': '8701.24', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 17:53:04,384 >> {'loss': 0.7584, 'grad_norm': 22.168949127197266, 'learning_rate': 2.528520199606434e-07, 'epoch': 0.09221589368049905, 'num_input_tokens_seen': 5746196480, 'completed': '74.32% (2_740 / 3_687)', 'remaining time': '8:18:13', 'throughput': '7584.46', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 17:53:36,716 >> {'loss': 0.6092, 'grad_norm': 20.419174194335938, 'learning_rate': 2.525489790534941e-07, 'epoch': 0.0924871168972064, 'num_input_tokens_seen': 5748293632, 'completed': '74.34% (2_741 / 3_687)', 'remaining time': '8:17:44', 'throughput': '8107.87', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 17:54:06,721 >> {'loss': 0.6697, 'grad_norm': 23.462411880493164, 'learning_rate': 2.5224617752360766e-07, 'epoch': 0.09275834011391375, 'num_input_tokens_seen': 5750390784, 'completed': '74.37% (2_742 / 3_687)', 'remaining time': '8:17:08', 'throughput': '8736.63', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 17:54:37,346 >> {'loss': 0.4207, 'grad_norm': 15.432395935058594, 'learning_rate': 2.519436156146671e-07, 'epoch': 0.0930295633306211, 'num_input_tokens_seen': 5752487936, 'completed': '74.40% (2_743 / 3_687)', 'remaining time': '8:16:34', 'throughput': '8559.85', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 17:55:06,918 >> {'loss': 0.5068, 'grad_norm': 17.80773162841797, 'learning_rate': 2.51641293570163e-07, 'epoch': 0.09330078654732846, 'num_input_tokens_seen': 5754585088, 'completed': '74.42% (2_744 / 3_687)', 'remaining time': '8:15:57', 'throughput': '8864.56', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 17:55:39,912 >> {'loss': 0.2518, 'grad_norm': 13.011022567749023, 'learning_rate': 2.513392116333922e-07, 'epoch': 0.0935720097640358, 'num_input_tokens_seen': 5756682240, 'completed': '74.45% (2_745 / 3_687)', 'remaining time': '8:15:29', 'throughput': '7945.22', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 17:56:12,662 >> {'loss': 0.4552, 'grad_norm': 16.054059982299805, 'learning_rate': 2.510373700474592e-07, 'epoch': 0.09384323298074315, 'num_input_tokens_seen': 5758779392, 'completed': '74.48% (2_746 / 3_687)', 'remaining time': '8:15:01', 'throughput': '8004.46', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 17:56:46,456 >> {'loss': 0.2891, 'grad_norm': 13.27901840209961, 'learning_rate': 2.5073576905527407e-07, 'epoch': 0.0941144561974505, 'num_input_tokens_seen': 5760876544, 'completed': '74.51% (2_747 / 3_687)', 'remaining time': '8:14:35', 'throughput': '7757.00', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 17:57:19,158 >> {'loss': 0.5392, 'grad_norm': 18.663637161254883, 'learning_rate': 2.5043440889955434e-07, 'epoch': 0.09438567941415785, 'num_input_tokens_seen': 5762973696, 'completed': '74.53% (2_748 / 3_687)', 'remaining time': '8:14:07', 'throughput': '8016.29', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 17:57:51,842 >> {'loss': 0.4573, 'grad_norm': 17.319080352783203, 'learning_rate': 2.501332898228232e-07, 'epoch': 0.09465690263086521, 'num_input_tokens_seen': 5765070848, 'completed': '74.56% (2_749 / 3_687)', 'remaining time': '8:13:38', 'throughput': '8020.58', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 17:58:23,418 >> {'loss': 0.3974, 'grad_norm': 17.932674407958984, 'learning_rate': 2.4983241206740945e-07, 'epoch': 0.09492812584757256, 'num_input_tokens_seen': 5767168000, 'completed': '74.59% (2_750 / 3_687)', 'remaining time': '8:13:06', 'throughput': '8301.81', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 17:58:55,031 >> {'loss': 0.5203, 'grad_norm': 16.8289737701416, 'learning_rate': 2.495317758754483e-07, 'epoch': 0.0951993490642799, 'num_input_tokens_seen': 5769265152, 'completed': '74.61% (2_751 / 3_687)', 'remaining time': '8:12:35', 'throughput': '8292.30', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 17:59:24,619 >> {'loss': 0.4871, 'grad_norm': 18.582918167114258, 'learning_rate': 2.492313814888802e-07, 'epoch': 0.09547057228098725, 'num_input_tokens_seen': 5771362304, 'completed': '74.64% (2_752 / 3_687)', 'remaining time': '8:11:58', 'throughput': '8859.98', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 17:59:55,997 >> {'loss': 0.3114, 'grad_norm': 13.084208488464355, 'learning_rate': 2.4893122914945124e-07, 'epoch': 0.0957417954976946, 'num_input_tokens_seen': 5773459456, 'completed': '74.67% (2_753 / 3_687)', 'remaining time': '8:11:26', 'throughput': '8354.37', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 18:00:23,545 >> {'loss': 0.4209, 'grad_norm': 23.180992126464844, 'learning_rate': 2.4863131909871247e-07, 'epoch': 0.09601301871440195, 'num_input_tokens_seen': 5775556608, 'completed': '74.69% (2_754 / 3_687)', 'remaining time': '8:10:44', 'throughput': '9515.89', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 18:00:53,515 >> {'loss': 0.5382, 'grad_norm': 19.62271499633789, 'learning_rate': 2.483316515780202e-07, 'epoch': 0.09628424193110931, 'num_input_tokens_seen': 5777653760, 'completed': '74.72% (2_755 / 3_687)', 'remaining time': '8:10:08', 'throughput': '8746.89', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 18:01:23,989 >> {'loss': 0.2602, 'grad_norm': 11.799657821655273, 'learning_rate': 2.480322268285354e-07, 'epoch': 0.09655546514781665, 'num_input_tokens_seen': 5779750912, 'completed': '74.75% (2_756 / 3_687)', 'remaining time': '8:09:34', 'throughput': '8602.22', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 18:01:55,289 >> {'loss': 0.3326, 'grad_norm': 14.03824234008789, 'learning_rate': 2.4773304509122374e-07, 'epoch': 0.096826688364524, 'num_input_tokens_seen': 5781848064, 'completed': '74.78% (2_757 / 3_687)', 'remaining time': '8:09:01', 'throughput': '8375.25', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 18:02:25,775 >> {'loss': 0.4274, 'grad_norm': 18.534460067749023, 'learning_rate': 2.474341066068553e-07, 'epoch': 0.09709791158123135, 'num_input_tokens_seen': 5783945216, 'completed': '74.80% (2_758 / 3_687)', 'remaining time': '8:08:27', 'throughput': '8598.64', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 18:02:56,914 >> {'loss': 0.681, 'grad_norm': 22.347103118896484, 'learning_rate': 2.4713541161600434e-07, 'epoch': 0.0973691347979387, 'num_input_tokens_seen': 5786042368, 'completed': '74.83% (2_759 / 3_687)', 'remaining time': '8:07:55', 'throughput': '8418.54', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 18:03:28,715 >> {'loss': 0.4139, 'grad_norm': 17.200597763061523, 'learning_rate': 2.4683696035904926e-07, 'epoch': 0.09764035801464606, 'num_input_tokens_seen': 5788139520, 'completed': '74.86% (2_760 / 3_687)', 'remaining time': '8:07:24', 'throughput': '8243.27', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 18:03:58,169 >> {'loss': 0.2853, 'grad_norm': 24.39967918395996, 'learning_rate': 2.465387530761724e-07, 'epoch': 0.0979115812313534, 'num_input_tokens_seen': 5790236672, 'completed': '74.88% (2_761 / 3_687)', 'remaining time': '8:06:47', 'throughput': '8900.24', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 18:04:29,305 >> {'loss': 0.3234, 'grad_norm': 15.185997009277344, 'learning_rate': 2.462407900073594e-07, 'epoch': 0.09818280444806075, 'num_input_tokens_seen': 5792333824, 'completed': '74.91% (2_762 / 3_687)', 'remaining time': '8:06:14', 'throughput': '8419.18', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 18:04:57,991 >> {'loss': 0.6504, 'grad_norm': 19.6497745513916, 'learning_rate': 2.459430713923995e-07, 'epoch': 0.0984540276647681, 'num_input_tokens_seen': 5794430976, 'completed': '74.94% (2_763 / 3_687)', 'remaining time': '8:05:35', 'throughput': '9138.32', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 18:05:29,654 >> {'loss': 0.2229, 'grad_norm': 11.902987480163574, 'learning_rate': 2.4564559747088573e-07, 'epoch': 0.09872525088147545, 'num_input_tokens_seen': 5796528128, 'completed': '74.97% (2_764 / 3_687)', 'remaining time': '8:05:04', 'throughput': '8279.38', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 18:05:59,983 >> {'loss': 0.4959, 'grad_norm': 34.26702880859375, 'learning_rate': 2.4534836848221355e-07, 'epoch': 0.09899647409818281, 'num_input_tokens_seen': 5798625280, 'completed': '74.99% (2_765 / 3_687)', 'remaining time': '8:04:30', 'throughput': '8643.14', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 18:06:29,296 >> {'loss': 0.4531, 'grad_norm': 16.864425659179688, 'learning_rate': 2.450513846655817e-07, 'epoch': 0.09926769731489016, 'num_input_tokens_seen': 5800722432, 'completed': '75.02% (2_766 / 3_687)', 'remaining time': '8:03:52', 'throughput': '8943.10', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 18:07:00,561 >> {'loss': 0.4869, 'grad_norm': 15.229866981506348, 'learning_rate': 2.4475464625999113e-07, 'epoch': 0.0995389205315975, 'num_input_tokens_seen': 5802819584, 'completed': '75.05% (2_767 / 3_687)', 'remaining time': '8:03:20', 'throughput': '8384.67', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 18:07:30,503 >> {'loss': 0.3666, 'grad_norm': 13.635716438293457, 'learning_rate': 2.444581535042459e-07, 'epoch': 0.09981014374830485, 'num_input_tokens_seen': 5804916736, 'completed': '75.07% (2_768 / 3_687)', 'remaining time': '8:02:45', 'throughput': '8754.99', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 18:07:59,331 >> {'loss': 0.9287, 'grad_norm': 24.428754806518555, 'learning_rate': 2.441619066369519e-07, 'epoch': 0.1000813669650122, 'num_input_tokens_seen': 5807013888, 'completed': '75.10% (2_769 / 3_687)', 'remaining time': '8:02:07', 'throughput': '9093.18', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 18:08:29,486 >> {'loss': 0.5323, 'grad_norm': 19.71752166748047, 'learning_rate': 2.438659058965175e-07, 'epoch': 0.10035259018171956, 'num_input_tokens_seen': 5809111040, 'completed': '75.13% (2_770 / 3_687)', 'remaining time': '8:01:32', 'throughput': '8693.27', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 18:08:59,885 >> {'loss': 0.3992, 'grad_norm': 17.558313369750977, 'learning_rate': 2.435701515211527e-07, 'epoch': 0.10062381339842691, 'num_input_tokens_seen': 5811208192, 'completed': '75.16% (2_771 / 3_687)', 'remaining time': '8:00:58', 'throughput': '8623.62', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 18:09:29,614 >> {'loss': 0.3217, 'grad_norm': 15.69237995147705, 'learning_rate': 2.4327464374886955e-07, 'epoch': 0.10089503661513426, 'num_input_tokens_seen': 5813305344, 'completed': '75.18% (2_772 / 3_687)', 'remaining time': '8:00:22', 'throughput': '8817.77', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 18:10:01,646 >> {'loss': 0.459, 'grad_norm': 20.842071533203125, 'learning_rate': 2.4297938281748124e-07, 'epoch': 0.1011662598318416, 'num_input_tokens_seen': 5815402496, 'completed': '75.21% (2_773 / 3_687)', 'remaining time': '7:59:51', 'throughput': '8183.63', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 18:10:35,297 >> {'loss': 0.5564, 'grad_norm': 19.98575782775879, 'learning_rate': 2.4268436896460267e-07, 'epoch': 0.10143748304854895, 'num_input_tokens_seen': 5817499648, 'completed': '75.24% (2_774 / 3_687)', 'remaining time': '7:59:25', 'throughput': '7790.57', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 18:11:02,943 >> {'loss': 0.6737, 'grad_norm': 18.332754135131836, 'learning_rate': 2.423896024276497e-07, 'epoch': 0.10170870626525631, 'num_input_tokens_seen': 5819596800, 'completed': '75.26% (2_775 / 3_687)', 'remaining time': '7:58:44', 'throughput': '9481.58', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 18:11:35,904 >> {'loss': 0.553, 'grad_norm': 22.827144622802734, 'learning_rate': 2.4209508344383926e-07, 'epoch': 0.10197992948196366, 'num_input_tokens_seen': 5821693952, 'completed': '75.29% (2_776 / 3_687)', 'remaining time': '7:58:16', 'throughput': '7953.08', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 18:12:04,781 >> {'loss': 0.6379, 'grad_norm': 21.232194900512695, 'learning_rate': 2.4180081225018906e-07, 'epoch': 0.10225115269867101, 'num_input_tokens_seen': 5823791104, 'completed': '75.32% (2_777 / 3_687)', 'remaining time': '7:57:39', 'throughput': '9077.98', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 18:12:33,553 >> {'loss': 0.7381, 'grad_norm': 19.167123794555664, 'learning_rate': 2.4150678908351744e-07, 'epoch': 0.10252237591537836, 'num_input_tokens_seen': 5825888256, 'completed': '75.35% (2_778 / 3_687)', 'remaining time': '7:57:00', 'throughput': '9111.04', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 18:13:05,701 >> {'loss': 0.3415, 'grad_norm': 15.559260368347168, 'learning_rate': 2.4121301418044264e-07, 'epoch': 0.1027935991320857, 'num_input_tokens_seen': 5827985408, 'completed': '75.37% (2_779 / 3_687)', 'remaining time': '7:56:31', 'throughput': '8154.47', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 18:13:38,719 >> {'loss': 0.274, 'grad_norm': 12.264792442321777, 'learning_rate': 2.409194877773839e-07, 'epoch': 0.10306482234879305, 'num_input_tokens_seen': 5830082560, 'completed': '75.40% (2_780 / 3_687)', 'remaining time': '7:56:03', 'throughput': '7939.36', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 18:14:08,781 >> {'loss': 0.4599, 'grad_norm': 17.90496826171875, 'learning_rate': 2.4062621011056006e-07, 'epoch': 0.10333604556550041, 'num_input_tokens_seen': 5832179712, 'completed': '75.43% (2_781 / 3_687)', 'remaining time': '7:55:28', 'throughput': '8720.05', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 18:14:39,447 >> {'loss': 0.3293, 'grad_norm': 12.02820110321045, 'learning_rate': 2.4033318141598977e-07, 'epoch': 0.10360726878220776, 'num_input_tokens_seen': 5834276864, 'completed': '75.45% (2_782 / 3_687)', 'remaining time': '7:54:54', 'throughput': '8548.38', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 18:15:09,316 >> {'loss': 0.4602, 'grad_norm': 15.775694847106934, 'learning_rate': 2.4004040192949155e-07, 'epoch': 0.1038784919989151, 'num_input_tokens_seen': 5836374016, 'completed': '75.48% (2_783 / 3_687)', 'remaining time': '7:54:19', 'throughput': '8776.50', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 18:15:41,623 >> {'loss': 0.3823, 'grad_norm': 18.71192741394043, 'learning_rate': 2.3974787188668284e-07, 'epoch': 0.10414971521562245, 'num_input_tokens_seen': 5838471168, 'completed': '75.51% (2_784 / 3_687)', 'remaining time': '7:53:50', 'throughput': '8114.09', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 18:16:10,598 >> {'loss': 0.5613, 'grad_norm': 20.695791244506836, 'learning_rate': 2.3945559152298085e-07, 'epoch': 0.1044209384323298, 'num_input_tokens_seen': 5840568320, 'completed': '75.54% (2_785 / 3_687)', 'remaining time': '7:53:12', 'throughput': '9047.41', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 18:16:42,921 >> {'loss': 0.3668, 'grad_norm': 14.568918228149414, 'learning_rate': 2.3916356107360174e-07, 'epoch': 0.10469216164903716, 'num_input_tokens_seen': 5842665472, 'completed': '75.56% (2_786 / 3_687)', 'remaining time': '7:52:43', 'throughput': '8109.93', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 18:17:13,728 >> {'loss': 0.518, 'grad_norm': 18.7004337310791, 'learning_rate': 2.388717807735605e-07, 'epoch': 0.10496338486574451, 'num_input_tokens_seen': 5844762624, 'completed': '75.59% (2_787 / 3_687)', 'remaining time': '7:52:10', 'throughput': '8509.39', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 18:17:45,508 >> {'loss': 0.2456, 'grad_norm': 11.687546730041504, 'learning_rate': 2.385802508576706e-07, 'epoch': 0.10523460808245186, 'num_input_tokens_seen': 5846859776, 'completed': '75.62% (2_788 / 3_687)', 'remaining time': '7:51:39', 'throughput': '8248.78', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 18:18:19,778 >> {'loss': 0.2584, 'grad_norm': 14.417143821716309, 'learning_rate': 2.382889715605447e-07, 'epoch': 0.1055058312991592, 'num_input_tokens_seen': 5848956928, 'completed': '75.64% (2_789 / 3_687)', 'remaining time': '7:51:14', 'throughput': '7649.25', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 18:18:50,384 >> {'loss': 0.2735, 'grad_norm': 14.506068229675293, 'learning_rate': 2.3799794311659286e-07, 'epoch': 0.10577705451586655, 'num_input_tokens_seen': 5851054080, 'completed': '75.67% (2_790 / 3_687)', 'remaining time': '7:50:40', 'throughput': '8565.33', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 18:19:22,342 >> {'loss': 0.5316, 'grad_norm': 30.506330490112305, 'learning_rate': 2.3770716576002383e-07, 'epoch': 0.10604827773257391, 'num_input_tokens_seen': 5853151232, 'completed': '75.70% (2_791 / 3_687)', 'remaining time': '7:50:10', 'throughput': '8202.76', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 18:19:51,492 >> {'loss': 0.5209, 'grad_norm': 18.837438583374023, 'learning_rate': 2.3741663972484416e-07, 'epoch': 0.10631950094928126, 'num_input_tokens_seen': 5855248384, 'completed': '75.73% (2_792 / 3_687)', 'remaining time': '7:49:33', 'throughput': '8992.73', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 18:20:21,917 >> {'loss': 0.2876, 'grad_norm': 13.7747163772583, 'learning_rate': 2.3712636524485817e-07, 'epoch': 0.10659072416598861, 'num_input_tokens_seen': 5857345536, 'completed': '75.75% (2_793 / 3_687)', 'remaining time': '7:48:59', 'throughput': '8616.25', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 18:20:51,666 >> {'loss': 0.2466, 'grad_norm': 13.69284439086914, 'learning_rate': 2.3683634255366773e-07, 'epoch': 0.10686194738269596, 'num_input_tokens_seen': 5859442688, 'completed': '75.78% (2_794 / 3_687)', 'remaining time': '7:48:24', 'throughput': '8811.87', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 18:21:19,182 >> {'loss': 0.5751, 'grad_norm': 23.49444007873535, 'learning_rate': 2.3654657188467203e-07, 'epoch': 0.1071331705994033, 'num_input_tokens_seen': 5861539840, 'completed': '75.81% (2_795 / 3_687)', 'remaining time': '7:47:43', 'throughput': '9526.86', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 18:21:51,632 >> {'loss': 0.4901, 'grad_norm': 32.023529052734375, 'learning_rate': 2.3625705347106748e-07, 'epoch': 0.10740439381611067, 'num_input_tokens_seen': 5863636992, 'completed': '75.83% (2_796 / 3_687)', 'remaining time': '7:47:14', 'throughput': '8078.47', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 18:22:22,652 >> {'loss': 0.4605, 'grad_norm': 18.634899139404297, 'learning_rate': 2.3596778754584752e-07, 'epoch': 0.10767561703281801, 'num_input_tokens_seen': 5865734144, 'completed': '75.86% (2_797 / 3_687)', 'remaining time': '7:46:42', 'throughput': '8450.76', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 18:22:54,990 >> {'loss': 0.4397, 'grad_norm': 17.947357177734375, 'learning_rate': 2.356787743418023e-07, 'epoch': 0.10794684024952536, 'num_input_tokens_seen': 5867831296, 'completed': '75.89% (2_798 / 3_687)', 'remaining time': '7:46:12', 'throughput': '8106.30', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 18:23:24,920 >> {'loss': 0.2498, 'grad_norm': 10.587057113647461, 'learning_rate': 2.3539001409151867e-07, 'epoch': 0.10821806346623271, 'num_input_tokens_seen': 5869928448, 'completed': '75.92% (2_799 / 3_687)', 'remaining time': '7:45:37', 'throughput': '8758.46', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 18:23:54,857 >> {'loss': 0.4346, 'grad_norm': 15.883231163024902, 'learning_rate': 2.351015070273799e-07, 'epoch': 0.10848928668294006, 'num_input_tokens_seen': 5872025600, 'completed': '75.94% (2_800 / 3_687)', 'remaining time': '7:45:03', 'throughput': '8756.70', 'gpu_mem_free': '5581MB'} +/scratch3/workspace/ctpham_umass_edu-ft/envs/prolong-final/lib/python3.10/site-packages/torch/distributed/fsdp/fully_sharded_data_parallel.py:689: FutureWarning: FSDP.state_dict_type() and FSDP.set_state_dict_type() are being deprecated. Please use APIs, get_state_dict() and set_state_dict(), which can support different parallelisms, FSDP1, FSDP2, DDP. API doc: https://pytorch.org/docs/stable/distributed.checkpoint.html#torch.distributed.checkpoint.state_dict.get_state_dict .Tutorial: https://pytorch.org/tutorials/recipes/distributed_checkpoint_recipe.html . + warnings.warn( +[INFO|trainer.py:3503] 2024-12-26 18:24:17,913 >> Saving model checkpoint to /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/checkpoint-2800 +[INFO|configuration_utils.py:472] 2024-12-26 18:24:17,918 >> Configuration saved in /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/checkpoint-2800/config.json +[INFO|configuration_utils.py:807] 2024-12-26 18:24:17,920 >> Configuration saved in /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/checkpoint-2800/generation_config.json +[INFO|modeling_utils.py:2807] 2024-12-26 18:25:14,975 >> The model is bigger than the maximum size per checkpoint (5GB) and is going to be split in 7 checkpoint shards. You can find where each parameters has been saved in the index located at /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/checkpoint-2800/model.safetensors.index.json. +[INFO|tokenization_utils_base.py:2684] 2024-12-26 18:25:14,978 >> tokenizer config file saved in /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/checkpoint-2800/tokenizer_config.json +[INFO|tokenization_utils_base.py:2693] 2024-12-26 18:25:14,979 >> Special tokens file saved in /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/checkpoint-2800/special_tokens_map.json +/scratch3/workspace/ctpham_umass_edu-ft/envs/prolong-final/lib/python3.10/site-packages/torch/distributed/fsdp/fully_sharded_data_parallel.py:689: FutureWarning: FSDP.state_dict_type() and FSDP.set_state_dict_type() are being deprecated. Please use APIs, get_state_dict() and set_state_dict(), which can support different parallelisms, FSDP1, FSDP2, DDP. API doc: https://pytorch.org/docs/stable/distributed.checkpoint.html#torch.distributed.checkpoint.state_dict.get_state_dict .Tutorial: https://pytorch.org/tutorials/recipes/distributed_checkpoint_recipe.html . + warnings.warn( +[WARNING|trainer.py:868] 2024-12-26 18:28:57,214 >> Save streaming dataset state: {'epoch': 0, 'sample_in_epoch': 5600, 'num_canonical_nodes': 1, 'shuffle_seed': 42, 'initial_physical_nodes': 1} +/scratch3/workspace/ctpham_umass_edu-ft/envs/prolong-final/lib/python3.10/site-packages/torch/utils/checkpoint.py:1399: FutureWarning: `torch.cpu.amp.autocast(args...)` is deprecated. Please use `torch.amp.autocast('cpu', args...)` instead. + with device_autocast_ctx, torch.cpu.amp.autocast(**cpu_autocast_kwargs), recompute_context: # type: ignore[attr-defined] +[INFO|trainer.py:175] 2024-12-26 18:29:31,102 >> {'loss': 0.7427, 'grad_norm': 22.052234649658203, 'learning_rate': 2.3481325338156568e-07, 'epoch': 0.1087605098996474, 'num_input_tokens_seen': 5874122752, 'completed': '75.97% (2_801 / 3_687)', 'remaining time': '7:55:44', 'throughput': '779.62', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 18:30:00,266 >> {'loss': 0.2588, 'grad_norm': 15.02530288696289, 'learning_rate': 2.3452525338605126e-07, 'epoch': 0.10903173311635476, 'num_input_tokens_seen': 5876219904, 'completed': '76.00% (2_802 / 3_687)', 'remaining time': '7:55:06', 'throughput': '8988.74', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 18:30:32,029 >> {'loss': 0.2948, 'grad_norm': 15.054925918579102, 'learning_rate': 2.3423750727260813e-07, 'epoch': 0.10930295633306211, 'num_input_tokens_seen': 5878317056, 'completed': '76.02% (2_803 / 3_687)', 'remaining time': '7:54:32', 'throughput': '8253.02', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 18:31:01,816 >> {'loss': 0.2361, 'grad_norm': 12.671406745910645, 'learning_rate': 2.3395001527280343e-07, 'epoch': 0.10957417954976946, 'num_input_tokens_seen': 5880414208, 'completed': '76.05% (2_804 / 3_687)', 'remaining time': '7:53:55', 'throughput': '8800.85', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 18:31:32,391 >> {'loss': 0.663, 'grad_norm': 20.93232536315918, 'learning_rate': 2.3366277761799993e-07, 'epoch': 0.10984540276647681, 'num_input_tokens_seen': 5882511360, 'completed': '76.08% (2_805 / 3_687)', 'remaining time': '7:53:19', 'throughput': '8573.53', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 18:32:04,969 >> {'loss': 0.5558, 'grad_norm': 18.05968475341797, 'learning_rate': 2.333757945393557e-07, 'epoch': 0.11011662598318415, 'num_input_tokens_seen': 5884608512, 'completed': '76.11% (2_806 / 3_687)', 'remaining time': '7:52:48', 'throughput': '8046.67', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 18:32:38,098 >> {'loss': 0.4106, 'grad_norm': 15.030449867248535, 'learning_rate': 2.3308906626782333e-07, 'epoch': 0.11038784919989152, 'num_input_tokens_seen': 5886705664, 'completed': '76.13% (2_807 / 3_687)', 'remaining time': '7:52:18', 'throughput': '7912.83', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 18:33:12,940 >> {'loss': 0.5442, 'grad_norm': 20.000356674194336, 'learning_rate': 2.3280259303415115e-07, 'epoch': 0.11065907241659886, 'num_input_tokens_seen': 5888802816, 'completed': '76.16% (2_808 / 3_687)', 'remaining time': '7:51:51', 'throughput': '7523.97', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 18:33:43,006 >> {'loss': 0.1988, 'grad_norm': 10.596423149108887, 'learning_rate': 2.3251637506888178e-07, 'epoch': 0.11093029563330621, 'num_input_tokens_seen': 5890899968, 'completed': '76.19% (2_809 / 3_687)', 'remaining time': '7:51:14', 'throughput': '8718.92', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 18:34:11,572 >> {'loss': 0.3359, 'grad_norm': 13.657599449157715, 'learning_rate': 2.3223041260235265e-07, 'epoch': 0.11120151885001356, 'num_input_tokens_seen': 5892997120, 'completed': '76.21% (2_810 / 3_687)', 'remaining time': '7:50:34', 'throughput': '9176.67', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 18:34:42,907 >> {'loss': 0.218, 'grad_norm': 12.095483779907227, 'learning_rate': 2.3194470586469543e-07, 'epoch': 0.1114727420667209, 'num_input_tokens_seen': 5895094272, 'completed': '76.24% (2_811 / 3_687)', 'remaining time': '7:50:00', 'throughput': '8365.98', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 18:35:12,547 >> {'loss': 0.3819, 'grad_norm': 16.444936752319336, 'learning_rate': 2.3165925508583598e-07, 'epoch': 0.11174396528342827, 'num_input_tokens_seen': 5897191424, 'completed': '76.27% (2_812 / 3_687)', 'remaining time': '7:49:23', 'throughput': '8844.15', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 18:35:43,628 >> {'loss': 0.5105, 'grad_norm': 20.455156326293945, 'learning_rate': 2.313740604954943e-07, 'epoch': 0.11201518850013562, 'num_input_tokens_seen': 5899288576, 'completed': '76.30% (2_813 / 3_687)', 'remaining time': '7:48:48', 'throughput': '8434.30', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 18:36:16,322 >> {'loss': 0.3829, 'grad_norm': 15.869861602783203, 'learning_rate': 2.3108912232318413e-07, 'epoch': 0.11228641171684296, 'num_input_tokens_seen': 5901385728, 'completed': '76.32% (2_814 / 3_687)', 'remaining time': '7:48:17', 'throughput': '8018.01', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 18:36:47,693 >> {'loss': 0.4095, 'grad_norm': 13.574760437011719, 'learning_rate': 2.3080444079821284e-07, 'epoch': 0.11255763493355031, 'num_input_tokens_seen': 5903482880, 'completed': '76.35% (2_815 / 3_687)', 'remaining time': '7:47:43', 'throughput': '8356.17', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 18:37:19,980 >> {'loss': 0.8766, 'grad_norm': 27.635112762451172, 'learning_rate': 2.305200161496813e-07, 'epoch': 0.11282885815025766, 'num_input_tokens_seen': 5905580032, 'completed': '76.38% (2_816 / 3_687)', 'remaining time': '7:47:11', 'throughput': '8119.33', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 18:37:48,792 >> {'loss': 0.6796, 'grad_norm': 20.717670440673828, 'learning_rate': 2.3023584860648364e-07, 'epoch': 0.11310008136696502, 'num_input_tokens_seen': 5907677184, 'completed': '76.40% (2_817 / 3_687)', 'remaining time': '7:46:32', 'throughput': '9098.27', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 18:38:19,693 >> {'loss': 0.469, 'grad_norm': 17.95741081237793, 'learning_rate': 2.2995193839730727e-07, 'epoch': 0.11337130458367237, 'num_input_tokens_seen': 5909774336, 'completed': '76.43% (2_818 / 3_687)', 'remaining time': '7:45:57', 'throughput': '8483.38', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 18:38:48,933 >> {'loss': 0.3544, 'grad_norm': 15.287071228027344, 'learning_rate': 2.2966828575063196e-07, 'epoch': 0.11364252780037971, 'num_input_tokens_seen': 5911871488, 'completed': '76.46% (2_819 / 3_687)', 'remaining time': '7:45:19', 'throughput': '8965.35', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 18:39:19,664 >> {'loss': 0.4254, 'grad_norm': 17.377723693847656, 'learning_rate': 2.2938489089473065e-07, 'epoch': 0.11391375101708706, 'num_input_tokens_seen': 5913968640, 'completed': '76.48% (2_820 / 3_687)', 'remaining time': '7:44:44', 'throughput': '8530.21', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 18:39:51,255 >> {'loss': 0.7153, 'grad_norm': 21.972118377685547, 'learning_rate': 2.2910175405766896e-07, 'epoch': 0.11418497423379441, 'num_input_tokens_seen': 5916065792, 'completed': '76.51% (2_821 / 3_687)', 'remaining time': '7:44:10', 'throughput': '8298.11', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 18:40:21,827 >> {'loss': 0.4351, 'grad_norm': 21.45004653930664, 'learning_rate': 2.288188754673045e-07, 'epoch': 0.11445619745050176, 'num_input_tokens_seen': 5918162944, 'completed': '76.54% (2_822 / 3_687)', 'remaining time': '7:43:35', 'throughput': '8574.52', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 18:40:52,063 >> {'loss': 0.6523, 'grad_norm': 20.235944747924805, 'learning_rate': 2.2853625535128735e-07, 'epoch': 0.11472742066720912, 'num_input_tokens_seen': 5920260096, 'completed': '76.57% (2_823 / 3_687)', 'remaining time': '7:42:59', 'throughput': '8670.11', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 18:41:22,605 >> {'loss': 0.2849, 'grad_norm': 12.907526016235352, 'learning_rate': 2.2825389393705922e-07, 'epoch': 0.11499864388391647, 'num_input_tokens_seen': 5922357248, 'completed': '76.59% (2_824 / 3_687)', 'remaining time': '7:42:23', 'throughput': '8583.00', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 18:41:54,885 >> {'loss': 0.5741, 'grad_norm': 19.846851348876953, 'learning_rate': 2.2797179145185384e-07, 'epoch': 0.11526986710062381, 'num_input_tokens_seen': 5924454400, 'completed': '76.62% (2_825 / 3_687)', 'remaining time': '7:41:52', 'throughput': '8120.94', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 18:42:27,929 >> {'loss': 0.2806, 'grad_norm': 14.583490371704102, 'learning_rate': 2.2768994812269666e-07, 'epoch': 0.11554109031733116, 'num_input_tokens_seen': 5926551552, 'completed': '76.65% (2_826 / 3_687)', 'remaining time': '7:41:21', 'throughput': '7933.14', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 18:43:02,206 >> {'loss': 0.3765, 'grad_norm': 16.440160751342773, 'learning_rate': 2.2740836417640432e-07, 'epoch': 0.11581231353403851, 'num_input_tokens_seen': 5928648704, 'completed': '76.67% (2_827 / 3_687)', 'remaining time': '7:40:53', 'throughput': '7647.84', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 18:43:33,541 >> {'loss': 0.6938, 'grad_norm': 22.054208755493164, 'learning_rate': 2.271270398395851e-07, 'epoch': 0.11608353675074587, 'num_input_tokens_seen': 5930745856, 'completed': '76.70% (2_828 / 3_687)', 'remaining time': '7:40:20', 'throughput': '8365.95', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 18:44:01,804 >> {'loss': 0.406, 'grad_norm': 13.32007122039795, 'learning_rate': 2.2684597533863793e-07, 'epoch': 0.11635475996745322, 'num_input_tokens_seen': 5932843008, 'completed': '76.73% (2_829 / 3_687)', 'remaining time': '7:39:40', 'throughput': '9275.04', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 18:44:34,218 >> {'loss': 0.2882, 'grad_norm': 13.135971069335938, 'learning_rate': 2.2656517089975286e-07, 'epoch': 0.11662598318416056, 'num_input_tokens_seen': 5934940160, 'completed': '76.76% (2_830 / 3_687)', 'remaining time': '7:39:08', 'throughput': '8087.37', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 18:45:05,186 >> {'loss': 0.4794, 'grad_norm': 23.185508728027344, 'learning_rate': 2.2628462674891062e-07, 'epoch': 0.11689720640086791, 'num_input_tokens_seen': 5937037312, 'completed': '76.78% (2_831 / 3_687)', 'remaining time': '7:38:33', 'throughput': '8464.94', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 18:45:34,348 >> {'loss': 0.4259, 'grad_norm': 16.763158798217773, 'learning_rate': 2.2600434311188253e-07, 'epoch': 0.11716842961757526, 'num_input_tokens_seen': 5939134464, 'completed': '76.81% (2_832 / 3_687)', 'remaining time': '7:37:55', 'throughput': '8989.21', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 18:46:07,055 >> {'loss': 0.5011, 'grad_norm': 28.8845272064209, 'learning_rate': 2.2572432021423018e-07, 'epoch': 0.11743965283428262, 'num_input_tokens_seen': 5941231616, 'completed': '76.84% (2_833 / 3_687)', 'remaining time': '7:37:24', 'throughput': '8015.09', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 18:46:34,006 >> {'loss': 0.6964, 'grad_norm': 21.761306762695312, 'learning_rate': 2.2544455828130528e-07, 'epoch': 0.11771087605098997, 'num_input_tokens_seen': 5943328768, 'completed': '76.86% (2_834 / 3_687)', 'remaining time': '7:36:42', 'throughput': '9726.40', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 18:47:06,145 >> {'loss': 0.5045, 'grad_norm': 18.940256118774414, 'learning_rate': 2.251650575382498e-07, 'epoch': 0.11798209926769732, 'num_input_tokens_seen': 5945425920, 'completed': '76.89% (2_835 / 3_687)', 'remaining time': '7:36:10', 'throughput': '8156.64', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 18:47:33,536 >> {'loss': 0.4833, 'grad_norm': 16.255231857299805, 'learning_rate': 2.2488581820999481e-07, 'epoch': 0.11825332248440466, 'num_input_tokens_seen': 5947523072, 'completed': '76.92% (2_836 / 3_687)', 'remaining time': '7:35:29', 'throughput': '9570.51', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 18:48:03,458 >> {'loss': 0.678, 'grad_norm': 19.86233139038086, 'learning_rate': 2.2460684052126195e-07, 'epoch': 0.11852454570111201, 'num_input_tokens_seen': 5949620224, 'completed': '76.95% (2_837 / 3_687)', 'remaining time': '7:34:52', 'throughput': '8760.84', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 18:48:31,465 >> {'loss': 0.5081, 'grad_norm': 17.203176498413086, 'learning_rate': 2.243281246965617e-07, 'epoch': 0.11879576891781937, 'num_input_tokens_seen': 5951717376, 'completed': '76.97% (2_838 / 3_687)', 'remaining time': '7:34:12', 'throughput': '9360.03', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 18:49:03,004 >> {'loss': 0.2694, 'grad_norm': 11.493597984313965, 'learning_rate': 2.24049670960194e-07, 'epoch': 0.11906699213452672, 'num_input_tokens_seen': 5953814528, 'completed': '77.00% (2_839 / 3_687)', 'remaining time': '7:33:39', 'throughput': '8311.83', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 18:49:31,947 >> {'loss': 0.3385, 'grad_norm': 15.567400932312012, 'learning_rate': 2.2377147953624776e-07, 'epoch': 0.11933821535123407, 'num_input_tokens_seen': 5955911680, 'completed': '77.03% (2_840 / 3_687)', 'remaining time': '7:33:01', 'throughput': '9057.28', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 18:50:01,155 >> {'loss': 0.5221, 'grad_norm': 20.829069137573242, 'learning_rate': 2.2349355064860104e-07, 'epoch': 0.11960943856794141, 'num_input_tokens_seen': 5958008832, 'completed': '77.05% (2_841 / 3_687)', 'remaining time': '7:32:23', 'throughput': '8974.98', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 18:50:32,793 >> {'loss': 0.6818, 'grad_norm': 25.062725067138672, 'learning_rate': 2.2321588452092006e-07, 'epoch': 0.11988066178464876, 'num_input_tokens_seen': 5960105984, 'completed': '77.08% (2_842 / 3_687)', 'remaining time': '7:31:50', 'throughput': '8285.73', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 18:51:03,908 >> {'loss': 0.2983, 'grad_norm': 12.187406539916992, 'learning_rate': 2.229384813766602e-07, 'epoch': 0.12015188500135611, 'num_input_tokens_seen': 5962203136, 'completed': '77.11% (2_843 / 3_687)', 'remaining time': '7:31:16', 'throughput': '8425.06', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 18:51:33,847 >> {'loss': 0.3099, 'grad_norm': 12.41606616973877, 'learning_rate': 2.2266134143906496e-07, 'epoch': 0.12042310821806347, 'num_input_tokens_seen': 5964300288, 'completed': '77.14% (2_844 / 3_687)', 'remaining time': '7:30:40', 'throughput': '8755.82', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 18:52:09,364 >> {'loss': 0.3705, 'grad_norm': 15.00600528717041, 'learning_rate': 2.2238446493116572e-07, 'epoch': 0.12069433143477082, 'num_input_tokens_seen': 5966397440, 'completed': '77.16% (2_845 / 3_687)', 'remaining time': '7:30:15', 'throughput': '7380.76', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 18:52:40,364 >> {'loss': 0.5625, 'grad_norm': 19.592304229736328, 'learning_rate': 2.2210785207578275e-07, 'epoch': 0.12096555465147817, 'num_input_tokens_seen': 5968494592, 'completed': '77.19% (2_846 / 3_687)', 'remaining time': '7:29:41', 'throughput': '8456.41', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 18:53:10,416 >> {'loss': 0.2395, 'grad_norm': 11.538047790527344, 'learning_rate': 2.2183150309552308e-07, 'epoch': 0.12123677786818551, 'num_input_tokens_seen': 5970591744, 'completed': '77.22% (2_847 / 3_687)', 'remaining time': '7:29:05', 'throughput': '8723.00', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 18:53:41,883 >> {'loss': 0.4497, 'grad_norm': 17.946090698242188, 'learning_rate': 2.2155541821278196e-07, 'epoch': 0.12150800108489286, 'num_input_tokens_seen': 5972688896, 'completed': '77.24% (2_848 / 3_687)', 'remaining time': '7:28:31', 'throughput': '8330.62', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 18:54:13,443 >> {'loss': 0.2957, 'grad_norm': 13.891773223876953, 'learning_rate': 2.2127959764974203e-07, 'epoch': 0.12177922430160022, 'num_input_tokens_seen': 5974786048, 'completed': '77.27% (2_849 / 3_687)', 'remaining time': '7:27:58', 'throughput': '8306.25', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 18:54:44,101 >> {'loss': 0.4703, 'grad_norm': 17.93677520751953, 'learning_rate': 2.2100404162837317e-07, 'epoch': 0.12205044751830757, 'num_input_tokens_seen': 5976883200, 'completed': '77.30% (2_850 / 3_687)', 'remaining time': '7:27:24', 'throughput': '8550.60', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 18:55:13,925 >> {'loss': 0.4836, 'grad_norm': 21.524486541748047, 'learning_rate': 2.2072875037043232e-07, 'epoch': 0.12232167073501492, 'num_input_tokens_seen': 5978980352, 'completed': '77.33% (2_851 / 3_687)', 'remaining time': '7:26:47', 'throughput': '8789.69', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 18:55:44,352 >> {'loss': 0.3129, 'grad_norm': 12.988191604614258, 'learning_rate': 2.204537240974636e-07, 'epoch': 0.12259289395172226, 'num_input_tokens_seen': 5981077504, 'completed': '77.35% (2_852 / 3_687)', 'remaining time': '7:26:12', 'throughput': '8615.60', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 18:56:19,415 >> {'loss': 0.5311, 'grad_norm': 20.595354080200195, 'learning_rate': 2.2017896303079743e-07, 'epoch': 0.12286411716842961, 'num_input_tokens_seen': 5983174656, 'completed': '77.38% (2_853 / 3_687)', 'remaining time': '7:25:46', 'throughput': '7476.24', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 18:56:51,543 >> {'loss': 0.4373, 'grad_norm': 22.666582107543945, 'learning_rate': 2.1990446739155128e-07, 'epoch': 0.12313534038513697, 'num_input_tokens_seen': 5985271808, 'completed': '77.41% (2_854 / 3_687)', 'remaining time': '7:25:14', 'throughput': '8159.41', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 18:57:23,739 >> {'loss': 0.2112, 'grad_norm': 11.09825611114502, 'learning_rate': 2.1963023740062864e-07, 'epoch': 0.12340656360184432, 'num_input_tokens_seen': 5987368960, 'completed': '77.43% (2_855 / 3_687)', 'remaining time': '7:24:42', 'throughput': '8142.20', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 18:57:55,716 >> {'loss': 0.2525, 'grad_norm': 11.390056610107422, 'learning_rate': 2.1935627327871948e-07, 'epoch': 0.12367778681855167, 'num_input_tokens_seen': 5989466112, 'completed': '77.46% (2_856 / 3_687)', 'remaining time': '7:24:10', 'throughput': '8198.01', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 18:58:25,786 >> {'loss': 0.4179, 'grad_norm': 12.461858749389648, 'learning_rate': 2.1908257524629963e-07, 'epoch': 0.12394901003525902, 'num_input_tokens_seen': 5991563264, 'completed': '77.49% (2_857 / 3_687)', 'remaining time': '7:23:34', 'throughput': '8717.54', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 18:58:59,853 >> {'loss': 0.5047, 'grad_norm': 17.467121124267578, 'learning_rate': 2.1880914352363106e-07, 'epoch': 0.12422023325196636, 'num_input_tokens_seen': 5993660416, 'completed': '77.52% (2_858 / 3_687)', 'remaining time': '7:23:06', 'throughput': '7694.94', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 18:59:27,186 >> {'loss': 0.5304, 'grad_norm': 18.427640914916992, 'learning_rate': 2.1853597833076088e-07, 'epoch': 0.12449145646867373, 'num_input_tokens_seen': 5995757568, 'completed': '77.54% (2_859 / 3_687)', 'remaining time': '7:22:25', 'throughput': '9590.93', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 18:59:56,849 >> {'loss': 0.4055, 'grad_norm': 13.977510452270508, 'learning_rate': 2.1826307988752212e-07, 'epoch': 0.12476267968538107, 'num_input_tokens_seen': 5997854720, 'completed': '77.57% (2_860 / 3_687)', 'remaining time': '7:21:49', 'throughput': '8837.52', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 19:00:26,866 >> {'loss': 0.3931, 'grad_norm': 15.963011741638184, 'learning_rate': 2.179904484135331e-07, 'epoch': 0.1250339029020884, 'num_input_tokens_seen': 5999951872, 'completed': '77.60% (2_861 / 3_687)', 'remaining time': '7:21:13', 'throughput': '8733.12', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 19:00:55,903 >> {'loss': 0.4642, 'grad_norm': 17.05299186706543, 'learning_rate': 2.177180841281974e-07, 'epoch': 0.12530512611879577, 'num_input_tokens_seen': 6002049024, 'completed': '77.62% (2_862 / 3_687)', 'remaining time': '7:20:35', 'throughput': '9027.74', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 19:01:25,831 >> {'loss': 0.425, 'grad_norm': 15.496081352233887, 'learning_rate': 2.1744598725070347e-07, 'epoch': 0.12557634933550313, 'num_input_tokens_seen': 6004146176, 'completed': '77.65% (2_863 / 3_687)', 'remaining time': '7:20:00', 'throughput': '8759.29', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 19:01:56,515 >> {'loss': 0.5045, 'grad_norm': 18.021099090576172, 'learning_rate': 2.1717415800002425e-07, 'epoch': 0.12584757255221046, 'num_input_tokens_seen': 6006243328, 'completed': '77.68% (2_864 / 3_687)', 'remaining time': '7:19:25', 'throughput': '8543.22', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 19:02:24,797 >> {'loss': 0.5096, 'grad_norm': 16.96639633178711, 'learning_rate': 2.1690259659491768e-07, 'epoch': 0.12611879576891782, 'num_input_tokens_seen': 6008340480, 'completed': '77.71% (2_865 / 3_687)', 'remaining time': '7:18:47', 'throughput': '9269.20', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 19:02:54,454 >> {'loss': 0.3753, 'grad_norm': 14.744760513305664, 'learning_rate': 2.166313032539261e-07, 'epoch': 0.12639001898562516, 'num_input_tokens_seen': 6010437632, 'completed': '77.73% (2_866 / 3_687)', 'remaining time': '7:18:10', 'throughput': '8839.08', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 19:03:21,949 >> {'loss': 0.4849, 'grad_norm': 16.412687301635742, 'learning_rate': 2.1636027819537605e-07, 'epoch': 0.12666124220233252, 'num_input_tokens_seen': 6012534784, 'completed': '77.76% (2_867 / 3_687)', 'remaining time': '7:17:30', 'throughput': '9534.35', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 19:03:54,229 >> {'loss': 0.2672, 'grad_norm': 14.01508617401123, 'learning_rate': 2.160895216373781e-07, 'epoch': 0.12693246541903988, 'num_input_tokens_seen': 6014631936, 'completed': '77.79% (2_868 / 3_687)', 'remaining time': '7:16:59', 'throughput': '8120.86', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 19:04:24,807 >> {'loss': 0.2023, 'grad_norm': 9.786606788635254, 'learning_rate': 2.15819033797827e-07, 'epoch': 0.12720368863574721, 'num_input_tokens_seen': 6016729088, 'completed': '77.81% (2_869 / 3_687)', 'remaining time': '7:16:24', 'throughput': '8572.84', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 19:04:55,462 >> {'loss': 0.4128, 'grad_norm': 15.194939613342285, 'learning_rate': 2.1554881489440092e-07, 'epoch': 0.12747491185245458, 'num_input_tokens_seen': 6018826240, 'completed': '77.84% (2_870 / 3_687)', 'remaining time': '7:15:50', 'throughput': '8551.66', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 19:05:25,466 >> {'loss': 0.5414, 'grad_norm': 24.29368019104004, 'learning_rate': 2.1527886514456178e-07, 'epoch': 0.1277461350691619, 'num_input_tokens_seen': 6020923392, 'completed': '77.87% (2_871 / 3_687)', 'remaining time': '7:15:14', 'throughput': '8736.83', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 19:05:55,799 >> {'loss': 0.3288, 'grad_norm': 14.61825180053711, 'learning_rate': 2.150091847655549e-07, 'epoch': 0.12801735828586927, 'num_input_tokens_seen': 6023020544, 'completed': '77.90% (2_872 / 3_687)', 'remaining time': '7:14:40', 'throughput': '8642.21', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 19:06:29,150 >> {'loss': 0.1956, 'grad_norm': 10.37586784362793, 'learning_rate': 2.1473977397440878e-07, 'epoch': 0.12828858150257663, 'num_input_tokens_seen': 6025117696, 'completed': '77.92% (2_873 / 3_687)', 'remaining time': '7:14:10', 'throughput': '7860.21', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 19:07:00,823 >> {'loss': 0.5343, 'grad_norm': 17.528772354125977, 'learning_rate': 2.14470632987935e-07, 'epoch': 0.12855980471928397, 'num_input_tokens_seen': 6027214848, 'completed': '77.95% (2_874 / 3_687)', 'remaining time': '7:13:37', 'throughput': '8276.60', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 19:07:31,595 >> {'loss': 0.193, 'grad_norm': 13.622241020202637, 'learning_rate': 2.1420176202272805e-07, 'epoch': 0.12883102793599133, 'num_input_tokens_seen': 6029312000, 'completed': '77.98% (2_875 / 3_687)', 'remaining time': '7:13:03', 'throughput': '8518.92', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 19:08:01,357 >> {'loss': 0.448, 'grad_norm': 17.937719345092773, 'learning_rate': 2.1393316129516482e-07, 'epoch': 0.12910225115269866, 'num_input_tokens_seen': 6031409152, 'completed': '78.00% (2_876 / 3_687)', 'remaining time': '7:12:27', 'throughput': '8807.81', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 19:08:35,938 >> {'loss': 0.3603, 'grad_norm': 19.698455810546875, 'learning_rate': 2.1366483102140497e-07, 'epoch': 0.12937347436940602, 'num_input_tokens_seen': 6033506304, 'completed': '78.03% (2_877 / 3_687)', 'remaining time': '7:12:00', 'throughput': '7580.62', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 19:09:06,362 >> {'loss': 0.3772, 'grad_norm': 15.03511905670166, 'learning_rate': 2.1339677141739074e-07, 'epoch': 0.12964469758611338, 'num_input_tokens_seen': 6035603456, 'completed': '78.06% (2_878 / 3_687)', 'remaining time': '7:11:25', 'throughput': '8616.31', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 19:09:35,607 >> {'loss': 0.5342, 'grad_norm': 17.960357666015625, 'learning_rate': 2.1312898269884616e-07, 'epoch': 0.12991592080282072, 'num_input_tokens_seen': 6037700608, 'completed': '78.09% (2_879 / 3_687)', 'remaining time': '7:10:48', 'throughput': '8963.76', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 19:10:07,282 >> {'loss': 0.2405, 'grad_norm': 12.057799339294434, 'learning_rate': 2.128614650812775e-07, 'epoch': 0.13018714401952808, 'num_input_tokens_seen': 6039797760, 'completed': '78.11% (2_880 / 3_687)', 'remaining time': '7:10:16', 'throughput': '8276.01', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 19:10:34,951 >> {'loss': 0.6078, 'grad_norm': 27.220239639282227, 'learning_rate': 2.125942187799726e-07, 'epoch': 0.1304583672362354, 'num_input_tokens_seen': 6041894912, 'completed': '78.14% (2_881 / 3_687)', 'remaining time': '7:09:37', 'throughput': '9474.39', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 19:11:04,133 >> {'loss': 0.7583, 'grad_norm': 27.639814376831055, 'learning_rate': 2.123272440100012e-07, 'epoch': 0.13072959045294277, 'num_input_tokens_seen': 6043992064, 'completed': '78.17% (2_882 / 3_687)', 'remaining time': '7:09:00', 'throughput': '8983.03', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 19:11:32,430 >> {'loss': 0.6885, 'grad_norm': 22.158348083496094, 'learning_rate': 2.1206054098621434e-07, 'epoch': 0.13100081366965013, 'num_input_tokens_seen': 6046089216, 'completed': '78.19% (2_883 / 3_687)', 'remaining time': '7:08:22', 'throughput': '9264.21', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 19:12:00,554 >> {'loss': 0.7587, 'grad_norm': 26.35175895690918, 'learning_rate': 2.117941099232446e-07, 'epoch': 0.13127203688635747, 'num_input_tokens_seen': 6048186368, 'completed': '78.22% (2_884 / 3_687)', 'remaining time': '7:07:44', 'throughput': '9320.83', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 19:12:33,875 >> {'loss': 0.2805, 'grad_norm': 14.008932113647461, 'learning_rate': 2.1152795103550547e-07, 'epoch': 0.13154326010306483, 'num_input_tokens_seen': 6050283520, 'completed': '78.25% (2_885 / 3_687)', 'remaining time': '7:07:14', 'throughput': '7867.33', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 19:13:02,363 >> {'loss': 0.6467, 'grad_norm': 21.59432029724121, 'learning_rate': 2.1126206453719157e-07, 'epoch': 0.13181448331977216, 'num_input_tokens_seen': 6052380672, 'completed': '78.28% (2_886 / 3_687)', 'remaining time': '7:06:36', 'throughput': '9201.79', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 19:13:34,334 >> {'loss': 0.3664, 'grad_norm': 16.01569938659668, 'learning_rate': 2.1099645064227817e-07, 'epoch': 0.13208570653647952, 'num_input_tokens_seen': 6054477824, 'completed': '78.30% (2_887 / 3_687)', 'remaining time': '7:06:04', 'throughput': '8199.55', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 19:14:03,763 >> {'loss': 0.2808, 'grad_norm': 15.261370658874512, 'learning_rate': 2.1073110956452126e-07, 'epoch': 0.13235692975318689, 'num_input_tokens_seen': 6056574976, 'completed': '78.33% (2_888 / 3_687)', 'remaining time': '7:05:28', 'throughput': '8907.71', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 19:14:33,084 >> {'loss': 0.3063, 'grad_norm': 14.421086311340332, 'learning_rate': 2.1046604151745723e-07, 'epoch': 0.13262815296989422, 'num_input_tokens_seen': 6058672128, 'completed': '78.36% (2_889 / 3_687)', 'remaining time': '7:04:52', 'throughput': '8940.32', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 19:15:03,289 >> {'loss': 0.3161, 'grad_norm': 15.327587127685547, 'learning_rate': 2.1020124671440274e-07, 'epoch': 0.13289937618660158, 'num_input_tokens_seen': 6060769280, 'completed': '78.38% (2_890 / 3_687)', 'remaining time': '7:04:17', 'throughput': '8678.77', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 19:15:31,065 >> {'loss': 0.3972, 'grad_norm': 18.59784507751465, 'learning_rate': 2.0993672536845458e-07, 'epoch': 0.13317059940330891, 'num_input_tokens_seen': 6062866432, 'completed': '78.41% (2_891 / 3_687)', 'remaining time': '7:03:38', 'throughput': '9437.86', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 19:16:00,771 >> {'loss': 0.4275, 'grad_norm': 16.773000717163086, 'learning_rate': 2.0967247769248968e-07, 'epoch': 0.13344182262001628, 'num_input_tokens_seen': 6064963584, 'completed': '78.44% (2_892 / 3_687)', 'remaining time': '7:03:03', 'throughput': '8824.60', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 19:16:32,133 >> {'loss': 0.2471, 'grad_norm': 14.18702220916748, 'learning_rate': 2.0940850389916398e-07, 'epoch': 0.13371304583672364, 'num_input_tokens_seen': 6067060736, 'completed': '78.46% (2_893 / 3_687)', 'remaining time': '7:02:30', 'throughput': '8358.68', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 19:17:01,047 >> {'loss': 0.176, 'grad_norm': 11.325303077697754, 'learning_rate': 2.09144804200914e-07, 'epoch': 0.13398426905343097, 'num_input_tokens_seen': 6069157888, 'completed': '78.49% (2_894 / 3_687)', 'remaining time': '7:01:53', 'throughput': '9066.42', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 19:17:31,656 >> {'loss': 0.2562, 'grad_norm': 13.870182037353516, 'learning_rate': 2.0888137880995514e-07, 'epoch': 0.13425549227013833, 'num_input_tokens_seen': 6071255040, 'completed': '78.52% (2_895 / 3_687)', 'remaining time': '7:01:19', 'throughput': '8564.24', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 19:18:00,591 >> {'loss': 0.5646, 'grad_norm': 21.369096755981445, 'learning_rate': 2.086182279382821e-07, 'epoch': 0.13452671548684567, 'num_input_tokens_seen': 6073352192, 'completed': '78.55% (2_896 / 3_687)', 'remaining time': '7:00:43', 'throughput': '9059.81', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 19:18:32,738 >> {'loss': 0.4004, 'grad_norm': 17.719070434570312, 'learning_rate': 2.083553517976686e-07, 'epoch': 0.13479793870355303, 'num_input_tokens_seen': 6075449344, 'completed': '78.57% (2_897 / 3_687)', 'remaining time': '7:00:11', 'throughput': '8154.46', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 19:19:00,311 >> {'loss': 0.4522, 'grad_norm': 16.208087921142578, 'learning_rate': 2.0809275059966764e-07, 'epoch': 0.13506916192026036, 'num_input_tokens_seen': 6077546496, 'completed': '78.60% (2_898 / 3_687)', 'remaining time': '6:59:32', 'throughput': '9507.11', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 19:19:29,282 >> {'loss': 0.2981, 'grad_norm': 11.811439514160156, 'learning_rate': 2.0783042455561023e-07, 'epoch': 0.13534038513696772, 'num_input_tokens_seen': 6079643648, 'completed': '78.63% (2_899 / 3_687)', 'remaining time': '6:58:56', 'throughput': '9048.53', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 19:19:59,900 >> {'loss': 0.3378, 'grad_norm': 14.665522575378418, 'learning_rate': 2.0756837387660676e-07, 'epoch': 0.13561160835367508, 'num_input_tokens_seen': 6081740800, 'completed': '78.65% (2_900 / 3_687)', 'remaining time': '6:58:22', 'throughput': '8561.97', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 19:20:29,510 >> {'loss': 0.3911, 'grad_norm': 18.123260498046875, 'learning_rate': 2.073065987735454e-07, 'epoch': 0.13588283157038242, 'num_input_tokens_seen': 6083837952, 'completed': '78.68% (2_901 / 3_687)', 'remaining time': '6:57:46', 'throughput': '8853.24', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 19:21:03,560 >> {'loss': 0.4768, 'grad_norm': 15.937180519104004, 'learning_rate': 2.070450994570928e-07, 'epoch': 0.13615405478708978, 'num_input_tokens_seen': 6085935104, 'completed': '78.71% (2_902 / 3_687)', 'remaining time': '6:57:18', 'throughput': '7698.78', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 19:21:34,393 >> {'loss': 0.4538, 'grad_norm': 18.534635543823242, 'learning_rate': 2.0678387613769397e-07, 'epoch': 0.1364252780037971, 'num_input_tokens_seen': 6088032256, 'completed': '78.74% (2_903 / 3_687)', 'remaining time': '6:56:44', 'throughput': '8501.92', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 19:22:05,097 >> {'loss': 0.5692, 'grad_norm': 19.72084617614746, 'learning_rate': 2.0652292902557117e-07, 'epoch': 0.13669650122050447, 'num_input_tokens_seen': 6090129408, 'completed': '78.76% (2_904 / 3_687)', 'remaining time': '6:56:11', 'throughput': '8537.69', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 19:22:32,669 >> {'loss': 0.7825, 'grad_norm': 23.22869110107422, 'learning_rate': 2.0626225833072487e-07, 'epoch': 0.13696772443721184, 'num_input_tokens_seen': 6092226560, 'completed': '78.79% (2_905 / 3_687)', 'remaining time': '6:55:32', 'throughput': '9507.66', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 19:23:03,522 >> {'loss': 0.3674, 'grad_norm': 16.263364791870117, 'learning_rate': 2.0600186426293282e-07, 'epoch': 0.13723894765391917, 'num_input_tokens_seen': 6094323712, 'completed': '78.82% (2_906 / 3_687)', 'remaining time': '6:54:59', 'throughput': '8496.57', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 19:23:34,527 >> {'loss': 0.2301, 'grad_norm': 15.390429496765137, 'learning_rate': 2.0574174703175034e-07, 'epoch': 0.13751017087062653, 'num_input_tokens_seen': 6096420864, 'completed': '78.84% (2_907 / 3_687)', 'remaining time': '6:54:25', 'throughput': '8455.06', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 19:24:04,719 >> {'loss': 0.4732, 'grad_norm': 19.23081398010254, 'learning_rate': 2.0548190684650981e-07, 'epoch': 0.13778139408733386, 'num_input_tokens_seen': 6098518016, 'completed': '78.87% (2_908 / 3_687)', 'remaining time': '6:53:51', 'throughput': '8682.56', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 19:24:34,651 >> {'loss': 0.5119, 'grad_norm': 16.774734497070312, 'learning_rate': 2.052223439163208e-07, 'epoch': 0.13805261730404123, 'num_input_tokens_seen': 6100615168, 'completed': '78.90% (2_909 / 3_687)', 'remaining time': '6:53:16', 'throughput': '8758.04', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 19:25:05,610 >> {'loss': 0.2929, 'grad_norm': 11.67673110961914, 'learning_rate': 2.049630584500696e-07, 'epoch': 0.1383238405207486, 'num_input_tokens_seen': 6102712320, 'completed': '78.93% (2_910 / 3_687)', 'remaining time': '6:52:43', 'throughput': '8467.41', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 19:25:37,512 >> {'loss': 0.5621, 'grad_norm': 22.138757705688477, 'learning_rate': 2.0470405065641938e-07, 'epoch': 0.13859506373745592, 'num_input_tokens_seen': 6104809472, 'completed': '78.95% (2_911 / 3_687)', 'remaining time': '6:52:11', 'throughput': '8216.97', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 19:26:11,496 >> {'loss': 0.5335, 'grad_norm': 18.90143585205078, 'learning_rate': 2.0444532074380973e-07, 'epoch': 0.13886628695416328, 'num_input_tokens_seen': 6106906624, 'completed': '78.98% (2_912 / 3_687)', 'remaining time': '6:51:42', 'throughput': '7713.94', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 19:26:43,528 >> {'loss': 0.237, 'grad_norm': 13.576014518737793, 'learning_rate': 2.0418686892045654e-07, 'epoch': 0.13913751017087062, 'num_input_tokens_seen': 6109003776, 'completed': '79.01% (2_913 / 3_687)', 'remaining time': '6:51:11', 'throughput': '8183.71', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 19:27:13,620 >> {'loss': 0.4888, 'grad_norm': 16.365047454833984, 'learning_rate': 2.039286953943521e-07, 'epoch': 0.13940873338757798, 'num_input_tokens_seen': 6111100928, 'completed': '79.03% (2_914 / 3_687)', 'remaining time': '6:50:36', 'throughput': '8711.57', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 19:27:47,281 >> {'loss': 0.3311, 'grad_norm': 15.33917236328125, 'learning_rate': 2.0367080037326472e-07, 'epoch': 0.13967995660428534, 'num_input_tokens_seen': 6113198080, 'completed': '79.06% (2_915 / 3_687)', 'remaining time': '6:50:07', 'throughput': '7787.56', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 19:28:19,379 >> {'loss': 0.3541, 'grad_norm': 14.788158416748047, 'learning_rate': 2.0341318406473833e-07, 'epoch': 0.13995117982099267, 'num_input_tokens_seen': 6115295232, 'completed': '79.09% (2_916 / 3_687)', 'remaining time': '6:49:35', 'throughput': '8167.04', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 19:28:52,124 >> {'loss': 0.4563, 'grad_norm': 18.039945602416992, 'learning_rate': 2.031558466760927e-07, 'epoch': 0.14022240303770003, 'num_input_tokens_seen': 6117392384, 'completed': '79.12% (2_917 / 3_687)', 'remaining time': '6:49:05', 'throughput': '8005.62', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 19:29:22,730 >> {'loss': 0.5077, 'grad_norm': 20.71190071105957, 'learning_rate': 2.0289878841442325e-07, 'epoch': 0.14049362625440737, 'num_input_tokens_seen': 6119489536, 'completed': '79.14% (2_918 / 3_687)', 'remaining time': '6:48:31', 'throughput': '8565.21', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 19:29:51,206 >> {'loss': 0.4001, 'grad_norm': 19.41553497314453, 'learning_rate': 2.0264200948660076e-07, 'epoch': 0.14076484947111473, 'num_input_tokens_seen': 6121586688, 'completed': '79.17% (2_919 / 3_687)', 'remaining time': '6:47:54', 'throughput': '9205.86', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 19:30:24,492 >> {'loss': 0.4873, 'grad_norm': 17.448558807373047, 'learning_rate': 2.0238551009927125e-07, 'epoch': 0.1410360726878221, 'num_input_tokens_seen': 6123683840, 'completed': '79.20% (2_920 / 3_687)', 'remaining time': '6:47:24', 'throughput': '7875.40', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 19:30:55,045 >> {'loss': 0.3583, 'grad_norm': 14.663477897644043, 'learning_rate': 2.021292904588553e-07, 'epoch': 0.14130729590452942, 'num_input_tokens_seen': 6125780992, 'completed': '79.22% (2_921 / 3_687)', 'remaining time': '6:46:50', 'throughput': '8580.08', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 19:31:27,821 >> {'loss': 0.4864, 'grad_norm': 19.37281608581543, 'learning_rate': 2.0187335077154905e-07, 'epoch': 0.14157851912123678, 'num_input_tokens_seen': 6127878144, 'completed': '79.25% (2_922 / 3_687)', 'remaining time': '6:46:20', 'throughput': '7998.03', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 19:31:55,873 >> {'loss': 0.5179, 'grad_norm': 16.566606521606445, 'learning_rate': 2.0161769124332278e-07, 'epoch': 0.14184974233794412, 'num_input_tokens_seen': 6129975296, 'completed': '79.28% (2_923 / 3_687)', 'remaining time': '6:45:42', 'throughput': '9344.93', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 19:32:27,244 >> {'loss': 0.3427, 'grad_norm': 14.152243614196777, 'learning_rate': 2.013623120799217e-07, 'epoch': 0.14212096555465148, 'num_input_tokens_seen': 6132072448, 'completed': '79.31% (2_924 / 3_687)', 'remaining time': '6:45:10', 'throughput': '8356.15', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 19:32:56,600 >> {'loss': 0.4143, 'grad_norm': 17.58519744873047, 'learning_rate': 2.0110721348686523e-07, 'epoch': 0.14239218877135884, 'num_input_tokens_seen': 6134169600, 'completed': '79.33% (2_925 / 3_687)', 'remaining time': '6:44:34', 'throughput': '8929.94', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 19:33:27,012 >> {'loss': 0.1861, 'grad_norm': 11.517704010009766, 'learning_rate': 2.0085239566944702e-07, 'epoch': 0.14266341198806617, 'num_input_tokens_seen': 6136266752, 'completed': '79.36% (2_926 / 3_687)', 'remaining time': '6:44:00', 'throughput': '8619.63', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 19:33:57,431 >> {'loss': 0.4736, 'grad_norm': 19.695545196533203, 'learning_rate': 2.0059785883273463e-07, 'epoch': 0.14293463520477354, 'num_input_tokens_seen': 6138363904, 'completed': '79.39% (2_927 / 3_687)', 'remaining time': '6:43:27', 'throughput': '8617.89', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 19:34:25,902 >> {'loss': 0.4326, 'grad_norm': 17.48176383972168, 'learning_rate': 2.003436031815698e-07, 'epoch': 0.14320585842148087, 'num_input_tokens_seen': 6140461056, 'completed': '79.41% (2_928 / 3_687)', 'remaining time': '6:42:50', 'throughput': '9207.38', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 19:34:58,495 >> {'loss': 0.3545, 'grad_norm': 15.76534652709961, 'learning_rate': 2.0008962892056762e-07, 'epoch': 0.14347708163818823, 'num_input_tokens_seen': 6142558208, 'completed': '79.44% (2_929 / 3_687)', 'remaining time': '6:42:19', 'throughput': '8043.00', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 19:35:31,461 >> {'loss': 0.4996, 'grad_norm': 16.841144561767578, 'learning_rate': 1.998359362541171e-07, 'epoch': 0.1437483048548956, 'num_input_tokens_seen': 6144655360, 'completed': '79.47% (2_930 / 3_687)', 'remaining time': '6:41:49', 'throughput': '7951.97', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 19:36:03,012 >> {'loss': 0.4388, 'grad_norm': 20.11952781677246, 'learning_rate': 1.9958252538638032e-07, 'epoch': 0.14401952807160293, 'num_input_tokens_seen': 6146752512, 'completed': '79.50% (2_931 / 3_687)', 'remaining time': '6:41:17', 'throughput': '8308.62', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 19:36:37,226 >> {'loss': 0.2301, 'grad_norm': 14.645740509033203, 'learning_rate': 1.9932939652129283e-07, 'epoch': 0.1442907512883103, 'num_input_tokens_seen': 6148849664, 'completed': '79.52% (2_932 / 3_687)', 'remaining time': '6:40:48', 'throughput': '7661.78', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 19:37:04,417 >> {'loss': 0.651, 'grad_norm': 20.628215789794922, 'learning_rate': 1.990765498625629e-07, 'epoch': 0.14456197450501762, 'num_input_tokens_seen': 6150946816, 'completed': '79.55% (2_933 / 3_687)', 'remaining time': '6:40:10', 'throughput': '9640.83', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 19:37:36,396 >> {'loss': 0.3269, 'grad_norm': 13.291131019592285, 'learning_rate': 1.988239856136719e-07, 'epoch': 0.14483319772172498, 'num_input_tokens_seen': 6153043968, 'completed': '79.58% (2_934 / 3_687)', 'remaining time': '6:39:38', 'throughput': '8197.49', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 19:38:05,632 >> {'loss': 0.4404, 'grad_norm': 16.319238662719727, 'learning_rate': 1.9857170397787415e-07, 'epoch': 0.14510442093843234, 'num_input_tokens_seen': 6155141120, 'completed': '79.60% (2_935 / 3_687)', 'remaining time': '6:39:02', 'throughput': '8966.33', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 19:38:35,815 >> {'loss': 0.3358, 'grad_norm': 14.976720809936523, 'learning_rate': 1.9831970515819625e-07, 'epoch': 0.14537564415513968, 'num_input_tokens_seen': 6157238272, 'completed': '79.63% (2_936 / 3_687)', 'remaining time': '6:38:28', 'throughput': '8685.15', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 19:39:08,846 >> {'loss': 0.2542, 'grad_norm': 10.872328758239746, 'learning_rate': 1.980679893574373e-07, 'epoch': 0.14564686737184704, 'num_input_tokens_seen': 6159335424, 'completed': '79.66% (2_937 / 3_687)', 'remaining time': '6:37:58', 'throughput': '7936.28', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 19:39:42,529 >> {'loss': 0.4478, 'grad_norm': 19.26199722290039, 'learning_rate': 1.9781655677816838e-07, 'epoch': 0.14591809058855437, 'num_input_tokens_seen': 6161432576, 'completed': '79.69% (2_938 / 3_687)', 'remaining time': '6:37:29', 'throughput': '7782.69', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 19:40:12,490 >> {'loss': 0.3474, 'grad_norm': 12.88478946685791, 'learning_rate': 1.97565407622733e-07, 'epoch': 0.14618931380526173, 'num_input_tokens_seen': 6163529728, 'completed': '79.71% (2_939 / 3_687)', 'remaining time': '6:36:54', 'throughput': '8749.56', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 19:40:40,896 >> {'loss': 0.6245, 'grad_norm': 21.489870071411133, 'learning_rate': 1.9731454209324644e-07, 'epoch': 0.14646053702196907, 'num_input_tokens_seen': 6165626880, 'completed': '79.74% (2_940 / 3_687)', 'remaining time': '6:36:18', 'throughput': '9228.41', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 19:41:11,158 >> {'loss': 0.5171, 'grad_norm': 18.655330657958984, 'learning_rate': 1.9706396039159568e-07, 'epoch': 0.14673176023867643, 'num_input_tokens_seen': 6167724032, 'completed': '79.77% (2_941 / 3_687)', 'remaining time': '6:35:44', 'throughput': '8662.57', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 19:41:45,419 >> {'loss': 0.309, 'grad_norm': 15.826581001281738, 'learning_rate': 1.9681366271943945e-07, 'epoch': 0.1470029834553838, 'num_input_tokens_seen': 6169821184, 'completed': '79.79% (2_942 / 3_687)', 'remaining time': '6:35:15', 'throughput': '7651.42', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 19:42:16,273 >> {'loss': 0.2265, 'grad_norm': 16.819740295410156, 'learning_rate': 1.965636492782077e-07, 'epoch': 0.14727420667209112, 'num_input_tokens_seen': 6171918336, 'completed': '79.82% (2_943 / 3_687)', 'remaining time': '6:34:42', 'throughput': '8496.12', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 19:42:47,644 >> {'loss': 0.8497, 'grad_norm': 27.977834701538086, 'learning_rate': 1.963139202691017e-07, 'epoch': 0.14754542988879848, 'num_input_tokens_seen': 6174015488, 'completed': '79.85% (2_944 / 3_687)', 'remaining time': '6:34:10', 'throughput': '8356.32', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 19:43:17,975 >> {'loss': 0.7042, 'grad_norm': 29.797565460205078, 'learning_rate': 1.9606447589309397e-07, 'epoch': 0.14781665310550582, 'num_input_tokens_seen': 6176112640, 'completed': '79.88% (2_945 / 3_687)', 'remaining time': '6:33:36', 'throughput': '8642.89', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 19:43:47,585 >> {'loss': 0.2392, 'grad_norm': 14.23031234741211, 'learning_rate': 1.9581531635092773e-07, 'epoch': 0.14808787632221318, 'num_input_tokens_seen': 6178209792, 'completed': '79.90% (2_946 / 3_687)', 'remaining time': '6:33:01', 'throughput': '8853.16', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 19:44:18,154 >> {'loss': 0.3154, 'grad_norm': 14.467376708984375, 'learning_rate': 1.9556644184311728e-07, 'epoch': 0.14835909953892054, 'num_input_tokens_seen': 6180306944, 'completed': '79.93% (2_947 / 3_687)', 'remaining time': '6:32:27', 'throughput': '8575.44', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 19:44:46,217 >> {'loss': 0.4227, 'grad_norm': 17.21703338623047, 'learning_rate': 1.9531785256994716e-07, 'epoch': 0.14863032275562787, 'num_input_tokens_seen': 6182404096, 'completed': '79.96% (2_948 / 3_687)', 'remaining time': '6:31:51', 'throughput': '9341.41', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 19:45:17,866 >> {'loss': 0.5356, 'grad_norm': 19.657089233398438, 'learning_rate': 1.9506954873147276e-07, 'epoch': 0.14890154597233524, 'num_input_tokens_seen': 6184501248, 'completed': '79.98% (2_949 / 3_687)', 'remaining time': '6:31:19', 'throughput': '8282.61', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 19:45:48,313 >> {'loss': 0.4391, 'grad_norm': 16.794424057006836, 'learning_rate': 1.9482153052751921e-07, 'epoch': 0.14917276918904257, 'num_input_tokens_seen': 6186598400, 'completed': '80.01% (2_950 / 3_687)', 'remaining time': '6:30:45', 'throughput': '8609.86', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 19:46:18,600 >> {'loss': 0.5088, 'grad_norm': 18.118986129760742, 'learning_rate': 1.9457379815768245e-07, 'epoch': 0.14944399240574993, 'num_input_tokens_seen': 6188695552, 'completed': '80.04% (2_951 / 3_687)', 'remaining time': '6:30:11', 'throughput': '8655.53', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 19:46:48,827 >> {'loss': 0.4736, 'grad_norm': 17.5322265625, 'learning_rate': 1.9432635182132795e-07, 'epoch': 0.1497152156224573, 'num_input_tokens_seen': 6190792704, 'completed': '80.07% (2_952 / 3_687)', 'remaining time': '6:29:37', 'throughput': '8672.35', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 19:47:19,244 >> {'loss': 0.3153, 'grad_norm': 15.88696575164795, 'learning_rate': 1.9407919171759112e-07, 'epoch': 0.14998643883916463, 'num_input_tokens_seen': 6192889856, 'completed': '80.09% (2_953 / 3_687)', 'remaining time': '6:29:03', 'throughput': '8618.32', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 19:47:49,526 >> {'loss': 0.4207, 'grad_norm': 30.527063369750977, 'learning_rate': 1.9383231804537704e-07, 'epoch': 0.150257662055872, 'num_input_tokens_seen': 6194987008, 'completed': '80.12% (2_954 / 3_687)', 'remaining time': '6:28:30', 'throughput': '8656.78', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 19:48:19,850 >> {'loss': 0.3108, 'grad_norm': 14.228474617004395, 'learning_rate': 1.9358573100336034e-07, 'epoch': 0.15052888527257932, 'num_input_tokens_seen': 6197084160, 'completed': '80.15% (2_955 / 3_687)', 'remaining time': '6:27:56', 'throughput': '8644.79', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 19:48:50,308 >> {'loss': 0.3895, 'grad_norm': 17.981746673583984, 'learning_rate': 1.933394307899846e-07, 'epoch': 0.15080010848928668, 'num_input_tokens_seen': 6199181312, 'completed': '80.17% (2_956 / 3_687)', 'remaining time': '6:27:22', 'throughput': '8606.74', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 19:49:21,039 >> {'loss': 0.7074, 'grad_norm': 27.902082443237305, 'learning_rate': 1.9309341760346304e-07, 'epoch': 0.15107133170599404, 'num_input_tokens_seen': 6201278464, 'completed': '80.20% (2_957 / 3_687)', 'remaining time': '6:26:49', 'throughput': '8530.23', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 19:49:51,459 >> {'loss': 0.2475, 'grad_norm': 14.604863166809082, 'learning_rate': 1.9284769164177767e-07, 'epoch': 0.15134255492270138, 'num_input_tokens_seen': 6203375616, 'completed': '80.23% (2_958 / 3_687)', 'remaining time': '6:26:16', 'throughput': '8617.62', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 19:50:21,406 >> {'loss': 0.4611, 'grad_norm': 23.595659255981445, 'learning_rate': 1.9260225310267925e-07, 'epoch': 0.15161377813940874, 'num_input_tokens_seen': 6205472768, 'completed': '80.25% (2_959 / 3_687)', 'remaining time': '6:25:41', 'throughput': '8753.54', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 19:50:51,611 >> {'loss': 0.5274, 'grad_norm': 20.186174392700195, 'learning_rate': 1.9235710218368784e-07, 'epoch': 0.15188500135611607, 'num_input_tokens_seen': 6207569920, 'completed': '80.28% (2_960 / 3_687)', 'remaining time': '6:25:08', 'throughput': '8678.82', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 19:51:21,747 >> {'loss': 0.4078, 'grad_norm': 15.750053405761719, 'learning_rate': 1.9211223908209114e-07, 'epoch': 0.15215622457282343, 'num_input_tokens_seen': 6209667072, 'completed': '80.31% (2_961 / 3_687)', 'remaining time': '6:24:34', 'throughput': '8698.71', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 19:51:52,284 >> {'loss': 0.3896, 'grad_norm': 17.78439712524414, 'learning_rate': 1.9186766399494581e-07, 'epoch': 0.1524274477895308, 'num_input_tokens_seen': 6211764224, 'completed': '80.34% (2_962 / 3_687)', 'remaining time': '6:24:00', 'throughput': '8584.41', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 19:52:22,370 >> {'loss': 0.3784, 'grad_norm': 15.00125503540039, 'learning_rate': 1.9162337711907657e-07, 'epoch': 0.15269867100623813, 'num_input_tokens_seen': 6213861376, 'completed': '80.36% (2_963 / 3_687)', 'remaining time': '6:23:26', 'throughput': '8713.29', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 19:52:54,120 >> {'loss': 0.1549, 'grad_norm': 10.61737060546875, 'learning_rate': 1.9137937865107606e-07, 'epoch': 0.1529698942229455, 'num_input_tokens_seen': 6215958528, 'completed': '80.39% (2_964 / 3_687)', 'remaining time': '6:22:54', 'throughput': '8256.34', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 19:53:24,209 >> {'loss': 0.2978, 'grad_norm': 15.642570495605469, 'learning_rate': 1.9113566878730515e-07, 'epoch': 0.15324111743965282, 'num_input_tokens_seen': 6218055680, 'completed': '80.42% (2_965 / 3_687)', 'remaining time': '6:22:21', 'throughput': '8712.48', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 19:53:53,645 >> {'loss': 0.9192, 'grad_norm': 24.957725524902344, 'learning_rate': 1.9089224772389223e-07, 'epoch': 0.15351234065636019, 'num_input_tokens_seen': 6220152832, 'completed': '80.44% (2_966 / 3_687)', 'remaining time': '6:21:46', 'throughput': '8905.47', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 19:54:26,441 >> {'loss': 0.2818, 'grad_norm': 12.823304176330566, 'learning_rate': 1.9064911565673328e-07, 'epoch': 0.15378356387306755, 'num_input_tokens_seen': 6222249984, 'completed': '80.47% (2_967 / 3_687)', 'remaining time': '6:21:15', 'throughput': '7993.10', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 19:54:59,358 >> {'loss': 0.4243, 'grad_norm': 18.57845687866211, 'learning_rate': 1.9040627278149168e-07, 'epoch': 0.15405478708977488, 'num_input_tokens_seen': 6224347136, 'completed': '80.50% (2_968 / 3_687)', 'remaining time': '6:20:45', 'throughput': '7963.74', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 19:55:27,239 >> {'loss': 0.901, 'grad_norm': 27.991865158081055, 'learning_rate': 1.9016371929359824e-07, 'epoch': 0.15432601030648224, 'num_input_tokens_seen': 6226444288, 'completed': '80.53% (2_969 / 3_687)', 'remaining time': '6:20:08', 'throughput': '9402.38', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 19:55:57,741 >> {'loss': 0.6081, 'grad_norm': 40.3478889465332, 'learning_rate': 1.8992145538825066e-07, 'epoch': 0.15459723352318958, 'num_input_tokens_seen': 6228541440, 'completed': '80.55% (2_970 / 3_687)', 'remaining time': '6:19:35', 'throughput': '8594.33', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 19:56:29,255 >> {'loss': 0.4356, 'grad_norm': 16.422760009765625, 'learning_rate': 1.8967948126041383e-07, 'epoch': 0.15486845673989694, 'num_input_tokens_seen': 6230638592, 'completed': '80.58% (2_971 / 3_687)', 'remaining time': '6:19:03', 'throughput': '8318.32', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 19:56:58,028 >> {'loss': 0.4776, 'grad_norm': 17.511781692504883, 'learning_rate': 1.8943779710481938e-07, 'epoch': 0.1551396799566043, 'num_input_tokens_seen': 6232735744, 'completed': '80.61% (2_972 / 3_687)', 'remaining time': '6:18:27', 'throughput': '9110.80', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 19:57:28,535 >> {'loss': 0.8041, 'grad_norm': 30.01336097717285, 'learning_rate': 1.891964031159653e-07, 'epoch': 0.15541090317331163, 'num_input_tokens_seen': 6234832896, 'completed': '80.63% (2_973 / 3_687)', 'remaining time': '6:17:54', 'throughput': '8592.94', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 19:58:00,225 >> {'loss': 0.5671, 'grad_norm': 21.783706665039062, 'learning_rate': 1.8895529948811638e-07, 'epoch': 0.155682126390019, 'num_input_tokens_seen': 6236930048, 'completed': '80.66% (2_974 / 3_687)', 'remaining time': '6:17:22', 'throughput': '8271.99', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 19:58:30,729 >> {'loss': 0.411, 'grad_norm': 20.175209045410156, 'learning_rate': 1.8871448641530353e-07, 'epoch': 0.15595334960672633, 'num_input_tokens_seen': 6239027200, 'completed': '80.69% (2_975 / 3_687)', 'remaining time': '6:16:49', 'throughput': '8593.90', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 19:59:01,076 >> {'loss': 0.4136, 'grad_norm': 14.450340270996094, 'learning_rate': 1.8847396409132423e-07, 'epoch': 0.1562245728234337, 'num_input_tokens_seen': 6241124352, 'completed': '80.72% (2_976 / 3_687)', 'remaining time': '6:16:15', 'throughput': '8638.18', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 19:59:31,471 >> {'loss': 0.4456, 'grad_norm': 16.498634338378906, 'learning_rate': 1.8823373270974164e-07, 'epoch': 0.15649579604014105, 'num_input_tokens_seen': 6243221504, 'completed': '80.74% (2_977 / 3_687)', 'remaining time': '6:15:42', 'throughput': '8624.47', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 20:00:02,923 >> {'loss': 0.2795, 'grad_norm': 16.770217895507812, 'learning_rate': 1.8799379246388463e-07, 'epoch': 0.15676701925684838, 'num_input_tokens_seen': 6245318656, 'completed': '80.77% (2_978 / 3_687)', 'remaining time': '6:15:10', 'throughput': '8334.74', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 20:00:31,555 >> {'loss': 0.8234, 'grad_norm': 25.7247371673584, 'learning_rate': 1.8775414354684804e-07, 'epoch': 0.15703824247355574, 'num_input_tokens_seen': 6247415808, 'completed': '80.80% (2_979 / 3_687)', 'remaining time': '6:14:34', 'throughput': '9155.77', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 20:01:03,210 >> {'loss': 0.1999, 'grad_norm': 19.52436065673828, 'learning_rate': 1.875147861514923e-07, 'epoch': 0.15730946569026308, 'num_input_tokens_seen': 6249512960, 'completed': '80.82% (2_980 / 3_687)', 'remaining time': '6:14:02', 'throughput': '8281.22', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 20:01:36,465 >> {'loss': 0.3077, 'grad_norm': 13.377266883850098, 'learning_rate': 1.8727572047044308e-07, 'epoch': 0.15758068890697044, 'num_input_tokens_seen': 6251610112, 'completed': '80.85% (2_981 / 3_687)', 'remaining time': '6:13:33', 'throughput': '7882.99', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 20:02:11,370 >> {'loss': 0.3646, 'grad_norm': 15.139887809753418, 'learning_rate': 1.8703694669609133e-07, 'epoch': 0.15785191212367777, 'num_input_tokens_seen': 6253707264, 'completed': '80.88% (2_982 / 3_687)', 'remaining time': '6:13:05', 'throughput': '7510.17', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 20:02:43,622 >> {'loss': 0.4268, 'grad_norm': 16.056617736816406, 'learning_rate': 1.8679846502059306e-07, 'epoch': 0.15812313534038513, 'num_input_tokens_seen': 6255804416, 'completed': '80.91% (2_983 / 3_687)', 'remaining time': '6:12:33', 'throughput': '8127.86', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 20:03:14,052 >> {'loss': 0.3879, 'grad_norm': 15.632225036621094, 'learning_rate': 1.8656027563586934e-07, 'epoch': 0.1583943585570925, 'num_input_tokens_seen': 6257901568, 'completed': '80.93% (2_984 / 3_687)', 'remaining time': '6:12:00', 'throughput': '8614.84', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 20:03:45,896 >> {'loss': 0.1322, 'grad_norm': 9.523789405822754, 'learning_rate': 1.863223787336059e-07, 'epoch': 0.15866558177379983, 'num_input_tokens_seen': 6259998720, 'completed': '80.96% (2_985 / 3_687)', 'remaining time': '6:11:28', 'throughput': '8231.96', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 20:04:18,348 >> {'loss': 0.5607, 'grad_norm': 19.751140594482422, 'learning_rate': 1.8608477450525308e-07, 'epoch': 0.1589368049905072, 'num_input_tokens_seen': 6262095872, 'completed': '80.99% (2_986 / 3_687)', 'remaining time': '6:10:58', 'throughput': '8078.07', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 20:04:51,833 >> {'loss': 0.2927, 'grad_norm': 13.901773452758789, 'learning_rate': 1.8584746314202574e-07, 'epoch': 0.15920802820721452, 'num_input_tokens_seen': 6264193024, 'completed': '81.01% (2_987 / 3_687)', 'remaining time': '6:10:28', 'throughput': '7828.56', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 20:05:20,359 >> {'loss': 0.5888, 'grad_norm': 19.8528995513916, 'learning_rate': 1.8561044483490301e-07, 'epoch': 0.1594792514239219, 'num_input_tokens_seen': 6266290176, 'completed': '81.04% (2_988 / 3_687)', 'remaining time': '6:09:52', 'throughput': '9189.73', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 20:05:48,702 >> {'loss': 0.4884, 'grad_norm': 15.824024200439453, 'learning_rate': 1.8537371977462835e-07, 'epoch': 0.15975047464062925, 'num_input_tokens_seen': 6268387328, 'completed': '81.07% (2_989 / 3_687)', 'remaining time': '6:09:16', 'throughput': '9248.95', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 20:06:18,721 >> {'loss': 0.6673, 'grad_norm': 23.57025718688965, 'learning_rate': 1.8513728815170885e-07, 'epoch': 0.16002169785733658, 'num_input_tokens_seen': 6270484480, 'completed': '81.10% (2_990 / 3_687)', 'remaining time': '6:08:43', 'throughput': '8732.58', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 20:06:50,792 >> {'loss': 0.546, 'grad_norm': 24.425518035888672, 'learning_rate': 1.8490115015641566e-07, 'epoch': 0.16029292107404394, 'num_input_tokens_seen': 6272581632, 'completed': '81.12% (2_991 / 3_687)', 'remaining time': '6:08:11', 'throughput': '8173.85', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 20:07:19,184 >> {'loss': 0.8573, 'grad_norm': 27.48299217224121, 'learning_rate': 1.846653059787839e-07, 'epoch': 0.16056414429075128, 'num_input_tokens_seen': 6274678784, 'completed': '81.15% (2_992 / 3_687)', 'remaining time': '6:07:36', 'throughput': '9233.07', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 20:07:48,649 >> {'loss': 0.7799, 'grad_norm': 26.690526962280273, 'learning_rate': 1.8442975580861185e-07, 'epoch': 0.16083536750745864, 'num_input_tokens_seen': 6276775936, 'completed': '81.18% (2_993 / 3_687)', 'remaining time': '6:07:01', 'throughput': '8896.87', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 20:08:24,138 >> {'loss': 0.3413, 'grad_norm': 17.080846786499023, 'learning_rate': 1.8419449983546136e-07, 'epoch': 0.161106590724166, 'num_input_tokens_seen': 6278873088, 'completed': '81.20% (2_994 / 3_687)', 'remaining time': '6:06:34', 'throughput': '7386.60', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 20:08:53,205 >> {'loss': 0.285, 'grad_norm': 17.67878532409668, 'learning_rate': 1.8395953824865762e-07, 'epoch': 0.16137781394087333, 'num_input_tokens_seen': 6280970240, 'completed': '81.23% (2_995 / 3_687)', 'remaining time': '6:05:59', 'throughput': '9018.56', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 20:09:23,842 >> {'loss': 0.468, 'grad_norm': 23.767698287963867, 'learning_rate': 1.8372487123728844e-07, 'epoch': 0.1616490371575807, 'num_input_tokens_seen': 6283067392, 'completed': '81.26% (2_996 / 3_687)', 'remaining time': '6:05:26', 'throughput': '8556.53', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 20:09:51,764 >> {'loss': 0.4408, 'grad_norm': 16.759511947631836, 'learning_rate': 1.8349049899020514e-07, 'epoch': 0.16192026037428803, 'num_input_tokens_seen': 6285164544, 'completed': '81.29% (2_997 / 3_687)', 'remaining time': '6:04:50', 'throughput': '9388.28', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 20:10:21,049 >> {'loss': 0.3345, 'grad_norm': 13.694295883178711, 'learning_rate': 1.832564216960215e-07, 'epoch': 0.1621914835909954, 'num_input_tokens_seen': 6287261696, 'completed': '81.31% (2_998 / 3_687)', 'remaining time': '6:04:15', 'throughput': '8951.63', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 20:10:50,254 >> {'loss': 0.3348, 'grad_norm': 15.297809600830078, 'learning_rate': 1.8302263954311408e-07, 'epoch': 0.16246270680770275, 'num_input_tokens_seen': 6289358848, 'completed': '81.34% (2_999 / 3_687)', 'remaining time': '6:03:41', 'throughput': '8975.92', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 20:11:19,341 >> {'loss': 0.7217, 'grad_norm': 20.984760284423828, 'learning_rate': 1.827891527196218e-07, 'epoch': 0.16273393002441008, 'num_input_tokens_seen': 6291456000, 'completed': '81.37% (3_000 / 3_687)', 'remaining time': '6:03:06', 'throughput': '9012.58', 'gpu_mem_free': '5581MB'} +/scratch3/workspace/ctpham_umass_edu-ft/envs/prolong-final/lib/python3.10/site-packages/torch/distributed/fsdp/fully_sharded_data_parallel.py:689: FutureWarning: FSDP.state_dict_type() and FSDP.set_state_dict_type() are being deprecated. Please use APIs, get_state_dict() and set_state_dict(), which can support different parallelisms, FSDP1, FSDP2, DDP. API doc: https://pytorch.org/docs/stable/distributed.checkpoint.html#torch.distributed.checkpoint.state_dict.get_state_dict .Tutorial: https://pytorch.org/tutorials/recipes/distributed_checkpoint_recipe.html . + warnings.warn( +[INFO|trainer.py:3503] 2024-12-26 20:11:41,851 >> Saving model checkpoint to /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/checkpoint-3000 +[INFO|configuration_utils.py:472] 2024-12-26 20:11:41,854 >> Configuration saved in /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/checkpoint-3000/config.json +[INFO|configuration_utils.py:807] 2024-12-26 20:11:41,855 >> Configuration saved in /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/checkpoint-3000/generation_config.json +[INFO|modeling_utils.py:2807] 2024-12-26 20:12:38,584 >> The model is bigger than the maximum size per checkpoint (5GB) and is going to be split in 7 checkpoint shards. You can find where each parameters has been saved in the index located at /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/checkpoint-3000/model.safetensors.index.json. +[INFO|tokenization_utils_base.py:2684] 2024-12-26 20:12:38,588 >> tokenizer config file saved in /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/checkpoint-3000/tokenizer_config.json +[INFO|tokenization_utils_base.py:2693] 2024-12-26 20:12:38,589 >> Special tokens file saved in /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/checkpoint-3000/special_tokens_map.json +/scratch3/workspace/ctpham_umass_edu-ft/envs/prolong-final/lib/python3.10/site-packages/torch/distributed/fsdp/fully_sharded_data_parallel.py:689: FutureWarning: FSDP.state_dict_type() and FSDP.set_state_dict_type() are being deprecated. Please use APIs, get_state_dict() and set_state_dict(), which can support different parallelisms, FSDP1, FSDP2, DDP. API doc: https://pytorch.org/docs/stable/distributed.checkpoint.html#torch.distributed.checkpoint.state_dict.get_state_dict .Tutorial: https://pytorch.org/tutorials/recipes/distributed_checkpoint_recipe.html . + warnings.warn( +[WARNING|trainer.py:868] 2024-12-26 20:16:14,925 >> Save streaming dataset state: {'epoch': 0, 'sample_in_epoch': 6000, 'num_canonical_nodes': 1, 'shuffle_seed': 42, 'initial_physical_nodes': 1} +/scratch3/workspace/ctpham_umass_edu-ft/envs/prolong-final/lib/python3.10/site-packages/torch/utils/checkpoint.py:1399: FutureWarning: `torch.cpu.amp.autocast(args...)` is deprecated. Please use `torch.amp.autocast('cpu', args...)` instead. + with device_autocast_ctx, torch.cpu.amp.autocast(**cpu_autocast_kwargs), recompute_context: # type: ignore[attr-defined] +[INFO|trainer.py:175] 2024-12-26 20:16:45,750 >> {'loss': 0.2591, 'grad_norm': 16.016870498657227, 'learning_rate': 1.8255596141344605e-07, 'epoch': 0.16300515324111745, 'num_input_tokens_seen': 6293553152, 'completed': '81.39% (3_001 / 3_687)', 'remaining time': '6:08:11', 'throughput': '803.11', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 20:17:18,819 >> {'loss': 0.4054, 'grad_norm': 15.872649192810059, 'learning_rate': 1.823230658122502e-07, 'epoch': 0.16327637645782478, 'num_input_tokens_seen': 6295650304, 'completed': '81.42% (3_002 / 3_687)', 'remaining time': '6:07:40', 'throughput': '7927.17', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 20:17:48,801 >> {'loss': 0.4199, 'grad_norm': 17.43962287902832, 'learning_rate': 1.820904661034599e-07, 'epoch': 0.16354759967453214, 'num_input_tokens_seen': 6297747456, 'completed': '81.45% (3_003 / 3_687)', 'remaining time': '6:07:05', 'throughput': '8743.47', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 20:18:19,459 >> {'loss': 0.3451, 'grad_norm': 16.44698715209961, 'learning_rate': 1.8185816247426245e-07, 'epoch': 0.1638188228912395, 'num_input_tokens_seen': 6299844608, 'completed': '81.48% (3_004 / 3_687)', 'remaining time': '6:06:31', 'throughput': '8550.63', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 20:18:53,436 >> {'loss': 0.6525, 'grad_norm': 22.08724021911621, 'learning_rate': 1.8162615511160701e-07, 'epoch': 0.16409004610794684, 'num_input_tokens_seen': 6301941760, 'completed': '81.50% (3_005 / 3_687)', 'remaining time': '6:06:01', 'throughput': '7715.16', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 20:19:25,744 >> {'loss': 0.297, 'grad_norm': 13.724467277526855, 'learning_rate': 1.8139444420220438e-07, 'epoch': 0.1643612693246542, 'num_input_tokens_seen': 6304038912, 'completed': '81.53% (3_006 / 3_687)', 'remaining time': '6:05:29', 'throughput': '8114.01', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 20:19:55,740 >> {'loss': 0.8337, 'grad_norm': 24.55242156982422, 'learning_rate': 1.8116302993252636e-07, 'epoch': 0.16463249254136153, 'num_input_tokens_seen': 6306136064, 'completed': '81.56% (3_007 / 3_687)', 'remaining time': '6:04:54', 'throughput': '8739.40', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 20:20:25,010 >> {'loss': 0.2984, 'grad_norm': 12.129997253417969, 'learning_rate': 1.809319124888067e-07, 'epoch': 0.1649037157580689, 'num_input_tokens_seen': 6308233216, 'completed': '81.58% (3_008 / 3_687)', 'remaining time': '6:04:18', 'throughput': '8955.89', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 20:20:55,710 >> {'loss': 0.2444, 'grad_norm': 12.80477523803711, 'learning_rate': 1.8070109205703972e-07, 'epoch': 0.16517493897477625, 'num_input_tokens_seen': 6310330368, 'completed': '81.61% (3_009 / 3_687)', 'remaining time': '6:03:45', 'throughput': '8539.04', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 20:21:26,455 >> {'loss': 0.5638, 'grad_norm': 21.404075622558594, 'learning_rate': 1.80470568822981e-07, 'epoch': 0.1654461621914836, 'num_input_tokens_seen': 6312427520, 'completed': '81.64% (3_010 / 3_687)', 'remaining time': '6:03:11', 'throughput': '8526.42', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 20:21:55,892 >> {'loss': 0.8146, 'grad_norm': 24.48054313659668, 'learning_rate': 1.8024034297214686e-07, 'epoch': 0.16571738540819095, 'num_input_tokens_seen': 6314524672, 'completed': '81.67% (3_011 / 3_687)', 'remaining time': '6:02:36', 'throughput': '8905.07', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 20:22:27,645 >> {'loss': 0.4131, 'grad_norm': 16.312585830688477, 'learning_rate': 1.8001041468981442e-07, 'epoch': 0.16598860862489828, 'num_input_tokens_seen': 6316621824, 'completed': '81.69% (3_012 / 3_687)', 'remaining time': '6:02:03', 'throughput': '8255.82', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 20:22:58,988 >> {'loss': 0.4172, 'grad_norm': 14.948566436767578, 'learning_rate': 1.79780784161021e-07, 'epoch': 0.16625983184160564, 'num_input_tokens_seen': 6318718976, 'completed': '81.72% (3_013 / 3_687)', 'remaining time': '6:01:30', 'throughput': '8363.62', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 20:23:28,739 >> {'loss': 0.6344, 'grad_norm': 23.79063606262207, 'learning_rate': 1.7955145157056456e-07, 'epoch': 0.166531055058313, 'num_input_tokens_seen': 6320816128, 'completed': '81.75% (3_014 / 3_687)', 'remaining time': '6:00:55', 'throughput': '8811.27', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 20:23:59,202 >> {'loss': 0.565, 'grad_norm': 19.02938461303711, 'learning_rate': 1.793224171030033e-07, 'epoch': 0.16680227827502034, 'num_input_tokens_seen': 6322913280, 'completed': '81.77% (3_015 / 3_687)', 'remaining time': '6:00:21', 'throughput': '8605.47', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 20:24:28,835 >> {'loss': 0.4183, 'grad_norm': 16.11639976501465, 'learning_rate': 1.7909368094265532e-07, 'epoch': 0.1670735014917277, 'num_input_tokens_seen': 6325010432, 'completed': '81.80% (3_016 / 3_687)', 'remaining time': '5:59:46', 'throughput': '8846.16', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 20:24:59,411 >> {'loss': 0.2774, 'grad_norm': 14.722761154174805, 'learning_rate': 1.788652432735991e-07, 'epoch': 0.16734472470843503, 'num_input_tokens_seen': 6327107584, 'completed': '81.83% (3_017 / 3_687)', 'remaining time': '5:59:12', 'throughput': '8573.67', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 20:25:29,634 >> {'loss': 0.2382, 'grad_norm': 11.639079093933105, 'learning_rate': 1.7863710427967222e-07, 'epoch': 0.1676159479251424, 'num_input_tokens_seen': 6329204736, 'completed': '81.86% (3_018 / 3_687)', 'remaining time': '5:58:38', 'throughput': '8673.61', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 20:26:04,296 >> {'loss': 0.5038, 'grad_norm': 18.875831604003906, 'learning_rate': 1.784092641444725e-07, 'epoch': 0.16788717114184976, 'num_input_tokens_seen': 6331301888, 'completed': '81.88% (3_019 / 3_687)', 'remaining time': '5:58:08', 'throughput': '7562.83', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 20:26:36,709 >> {'loss': 0.5799, 'grad_norm': 21.543594360351562, 'learning_rate': 1.7818172305135683e-07, 'epoch': 0.1681583943585571, 'num_input_tokens_seen': 6333399040, 'completed': '81.91% (3_020 / 3_687)', 'remaining time': '5:57:36', 'throughput': '8087.59', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 20:27:04,377 >> {'loss': 0.4241, 'grad_norm': 17.74955177307129, 'learning_rate': 1.7795448118344174e-07, 'epoch': 0.16842961757526445, 'num_input_tokens_seen': 6335496192, 'completed': '81.94% (3_021 / 3_687)', 'remaining time': '5:56:59', 'throughput': '9474.78', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 20:27:35,683 >> {'loss': 0.4974, 'grad_norm': 18.136749267578125, 'learning_rate': 1.7772753872360286e-07, 'epoch': 0.16870084079197178, 'num_input_tokens_seen': 6337593344, 'completed': '81.96% (3_022 / 3_687)', 'remaining time': '5:56:26', 'throughput': '8373.53', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 20:28:11,427 >> {'loss': 0.4158, 'grad_norm': 16.745140075683594, 'learning_rate': 1.7750089585447473e-07, 'epoch': 0.16897206400867915, 'num_input_tokens_seen': 6339690496, 'completed': '81.99% (3_023 / 3_687)', 'remaining time': '5:55:58', 'throughput': '7333.96', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 20:28:41,573 >> {'loss': 0.3088, 'grad_norm': 13.123845100402832, 'learning_rate': 1.77274552758451e-07, 'epoch': 0.16924328722538648, 'num_input_tokens_seen': 6341787648, 'completed': '82.02% (3_024 / 3_687)', 'remaining time': '5:55:24', 'throughput': '8695.81', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 20:29:14,265 >> {'loss': 0.351, 'grad_norm': 13.37322998046875, 'learning_rate': 1.770485096176839e-07, 'epoch': 0.16951451044209384, 'num_input_tokens_seen': 6343884800, 'completed': '82.05% (3_025 / 3_687)', 'remaining time': '5:54:52', 'throughput': '8018.56', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 20:29:42,851 >> {'loss': 0.6836, 'grad_norm': 20.165077209472656, 'learning_rate': 1.7682276661408448e-07, 'epoch': 0.1697857336588012, 'num_input_tokens_seen': 6345981952, 'completed': '82.07% (3_026 / 3_687)', 'remaining time': '5:54:16', 'throughput': '9170.50', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 20:30:14,072 >> {'loss': 0.4345, 'grad_norm': 15.843926429748535, 'learning_rate': 1.7659732392932202e-07, 'epoch': 0.17005695687550854, 'num_input_tokens_seen': 6348079104, 'completed': '82.10% (3_027 / 3_687)', 'remaining time': '5:53:43', 'throughput': '8396.43', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 20:30:46,788 >> {'loss': 0.3054, 'grad_norm': 12.77824878692627, 'learning_rate': 1.7637218174482422e-07, 'epoch': 0.1703281800922159, 'num_input_tokens_seen': 6350176256, 'completed': '82.13% (3_028 / 3_687)', 'remaining time': '5:53:11', 'throughput': '8012.74', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 20:31:17,865 >> {'loss': 0.4641, 'grad_norm': 15.227442741394043, 'learning_rate': 1.7614734024177707e-07, 'epoch': 0.17059940330892323, 'num_input_tokens_seen': 6352273408, 'completed': '82.15% (3_029 / 3_687)', 'remaining time': '5:52:38', 'throughput': '8435.18', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 20:31:47,511 >> {'loss': 0.649, 'grad_norm': 20.998775482177734, 'learning_rate': 1.7592279960112424e-07, 'epoch': 0.1708706265256306, 'num_input_tokens_seen': 6354370560, 'completed': '82.18% (3_030 / 3_687)', 'remaining time': '5:52:03', 'throughput': '8842.42', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 20:32:19,201 >> {'loss': 0.3258, 'grad_norm': 14.684011459350586, 'learning_rate': 1.7569856000356758e-07, 'epoch': 0.17114184974233795, 'num_input_tokens_seen': 6356467712, 'completed': '82.21% (3_031 / 3_687)', 'remaining time': '5:51:31', 'throughput': '8272.04', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 20:32:48,979 >> {'loss': 0.6493, 'grad_norm': 22.120227813720703, 'learning_rate': 1.754746216295665e-07, 'epoch': 0.1714130729590453, 'num_input_tokens_seen': 6358564864, 'completed': '82.23% (3_032 / 3_687)', 'remaining time': '5:50:56', 'throughput': '8803.42', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 20:33:19,451 >> {'loss': 0.6875, 'grad_norm': 25.33750343322754, 'learning_rate': 1.7525098465933834e-07, 'epoch': 0.17168429617575265, 'num_input_tokens_seen': 6360662016, 'completed': '82.26% (3_033 / 3_687)', 'remaining time': '5:50:22', 'throughput': '8602.65', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 20:33:50,059 >> {'loss': 0.2934, 'grad_norm': 14.21183967590332, 'learning_rate': 1.750276492728575e-07, 'epoch': 0.17195551939245998, 'num_input_tokens_seen': 6362759168, 'completed': '82.29% (3_034 / 3_687)', 'remaining time': '5:49:49', 'throughput': '8564.82', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 20:34:21,806 >> {'loss': 0.337, 'grad_norm': 13.687496185302734, 'learning_rate': 1.748046156498557e-07, 'epoch': 0.17222674260916734, 'num_input_tokens_seen': 6364856320, 'completed': '82.32% (3_035 / 3_687)', 'remaining time': '5:49:16', 'throughput': '8257.26', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 20:34:51,432 >> {'loss': 0.5263, 'grad_norm': 16.938383102416992, 'learning_rate': 1.74581883969822e-07, 'epoch': 0.1724979658258747, 'num_input_tokens_seen': 6366953472, 'completed': '82.34% (3_036 / 3_687)', 'remaining time': '5:48:41', 'throughput': '8848.33', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 20:35:20,050 >> {'loss': 0.4494, 'grad_norm': 15.311517715454102, 'learning_rate': 1.7435945441200232e-07, 'epoch': 0.17276918904258204, 'num_input_tokens_seen': 6369050624, 'completed': '82.37% (3_037 / 3_687)', 'remaining time': '5:48:06', 'throughput': '9160.16', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 20:35:50,030 >> {'loss': 0.3876, 'grad_norm': 15.489421844482422, 'learning_rate': 1.7413732715539954e-07, 'epoch': 0.1730404122592894, 'num_input_tokens_seen': 6371147776, 'completed': '82.40% (3_038 / 3_687)', 'remaining time': '5:47:31', 'throughput': '8743.77', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 20:36:19,298 >> {'loss': 0.6567, 'grad_norm': 21.3696231842041, 'learning_rate': 1.7391550237877326e-07, 'epoch': 0.17331163547599673, 'num_input_tokens_seen': 6373244928, 'completed': '82.42% (3_039 / 3_687)', 'remaining time': '5:46:56', 'throughput': '8956.88', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 20:36:46,506 >> {'loss': 0.4457, 'grad_norm': 16.473154067993164, 'learning_rate': 1.7369398026063958e-07, 'epoch': 0.1735828586927041, 'num_input_tokens_seen': 6375342080, 'completed': '82.45% (3_040 / 3_687)', 'remaining time': '5:46:19', 'throughput': '9634.83', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 20:37:14,600 >> {'loss': 0.6155, 'grad_norm': 17.838611602783203, 'learning_rate': 1.7347276097927105e-07, 'epoch': 0.17385408190941146, 'num_input_tokens_seen': 6377439232, 'completed': '82.48% (3_041 / 3_687)', 'remaining time': '5:45:43', 'throughput': '9330.85', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 20:37:48,680 >> {'loss': 0.4533, 'grad_norm': 17.602170944213867, 'learning_rate': 1.732518447126966e-07, 'epoch': 0.1741253051261188, 'num_input_tokens_seen': 6379536384, 'completed': '82.51% (3_042 / 3_687)', 'remaining time': '5:45:13', 'throughput': '7692.06', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 20:38:17,794 >> {'loss': 0.4754, 'grad_norm': 25.11285400390625, 'learning_rate': 1.730312316387012e-07, 'epoch': 0.17439652834282615, 'num_input_tokens_seen': 6381633536, 'completed': '82.53% (3_043 / 3_687)', 'remaining time': '5:44:38', 'throughput': '9003.89', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 20:38:47,169 >> {'loss': 0.4263, 'grad_norm': 15.209177017211914, 'learning_rate': 1.7281092193482582e-07, 'epoch': 0.17466775155953349, 'num_input_tokens_seen': 6383730688, 'completed': '82.56% (3_044 / 3_687)', 'remaining time': '5:44:03', 'throughput': '8924.05', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 20:39:17,918 >> {'loss': 0.5498, 'grad_norm': 27.183820724487305, 'learning_rate': 1.7259091577836737e-07, 'epoch': 0.17493897477624085, 'num_input_tokens_seen': 6385827840, 'completed': '82.59% (3_045 / 3_687)', 'remaining time': '5:43:29', 'throughput': '8525.53', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 20:39:45,895 >> {'loss': 0.5436, 'grad_norm': 17.119733810424805, 'learning_rate': 1.723712133463785e-07, 'epoch': 0.1752101979929482, 'num_input_tokens_seen': 6387924992, 'completed': '82.61% (3_046 / 3_687)', 'remaining time': '5:42:53', 'throughput': '9369.91', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 20:40:15,799 >> {'loss': 0.2955, 'grad_norm': 14.52611255645752, 'learning_rate': 1.7215181481566727e-07, 'epoch': 0.17548142120965554, 'num_input_tokens_seen': 6390022144, 'completed': '82.64% (3_047 / 3_687)', 'remaining time': '5:42:19', 'throughput': '8766.12', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 20:40:49,063 >> {'loss': 0.2517, 'grad_norm': 11.77927303314209, 'learning_rate': 1.719327203627971e-07, 'epoch': 0.1757526444263629, 'num_input_tokens_seen': 6392119296, 'completed': '82.67% (3_048 / 3_687)', 'remaining time': '5:41:48', 'throughput': '7880.75', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 20:41:19,507 >> {'loss': 0.3709, 'grad_norm': 16.277713775634766, 'learning_rate': 1.7171393016408715e-07, 'epoch': 0.17602386764307024, 'num_input_tokens_seen': 6394216448, 'completed': '82.70% (3_049 / 3_687)', 'remaining time': '5:41:14', 'throughput': '8610.53', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 20:41:52,237 >> {'loss': 0.3434, 'grad_norm': 18.34269142150879, 'learning_rate': 1.7149544439561135e-07, 'epoch': 0.1762950908597776, 'num_input_tokens_seen': 6396313600, 'completed': '82.72% (3_050 / 3_687)', 'remaining time': '5:40:43', 'throughput': '8009.51', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 20:42:23,425 >> {'loss': 0.3524, 'grad_norm': 15.846407890319824, 'learning_rate': 1.7127726323319874e-07, 'epoch': 0.17656631407648496, 'num_input_tokens_seen': 6398410752, 'completed': '82.75% (3_051 / 3_687)', 'remaining time': '5:40:10', 'throughput': '8405.26', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 20:42:54,820 >> {'loss': 0.2759, 'grad_norm': 13.641855239868164, 'learning_rate': 1.7105938685243339e-07, 'epoch': 0.1768375372931923, 'num_input_tokens_seen': 6400507904, 'completed': '82.78% (3_052 / 3_687)', 'remaining time': '5:39:37', 'throughput': '8349.72', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 20:43:24,277 >> {'loss': 0.506, 'grad_norm': 16.64573097229004, 'learning_rate': 1.7084181542865355e-07, 'epoch': 0.17710876050989965, 'num_input_tokens_seen': 6402605056, 'completed': '82.80% (3_053 / 3_687)', 'remaining time': '5:39:02', 'throughput': '8899.14', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 20:43:53,896 >> {'loss': 0.5754, 'grad_norm': 21.56814193725586, 'learning_rate': 1.7062454913695264e-07, 'epoch': 0.177379983726607, 'num_input_tokens_seen': 6404702208, 'completed': '82.83% (3_054 / 3_687)', 'remaining time': '5:38:28', 'throughput': '8850.63', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 20:44:24,071 >> {'loss': 0.3337, 'grad_norm': 13.835966110229492, 'learning_rate': 1.7040758815217827e-07, 'epoch': 0.17765120694331435, 'num_input_tokens_seen': 6406799360, 'completed': '82.86% (3_055 / 3_687)', 'remaining time': '5:37:54', 'throughput': '8687.40', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 20:44:54,261 >> {'loss': 0.625, 'grad_norm': 19.247650146484375, 'learning_rate': 1.7019093264893236e-07, 'epoch': 0.1779224301600217, 'num_input_tokens_seen': 6408896512, 'completed': '82.89% (3_056 / 3_687)', 'remaining time': '5:37:20', 'throughput': '8683.21', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 20:45:22,436 >> {'loss': 0.3558, 'grad_norm': 13.838505744934082, 'learning_rate': 1.6997458280157113e-07, 'epoch': 0.17819365337672904, 'num_input_tokens_seen': 6410993664, 'completed': '82.91% (3_057 / 3_687)', 'remaining time': '5:36:44', 'throughput': '9304.14', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 20:45:51,328 >> {'loss': 0.5823, 'grad_norm': 19.8231143951416, 'learning_rate': 1.6975853878420457e-07, 'epoch': 0.1784648765934364, 'num_input_tokens_seen': 6413090816, 'completed': '82.94% (3_058 / 3_687)', 'remaining time': '5:36:09', 'throughput': '9073.38', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 20:46:22,965 >> {'loss': 0.3463, 'grad_norm': 16.521181106567383, 'learning_rate': 1.6954280077069676e-07, 'epoch': 0.17873609981014374, 'num_input_tokens_seen': 6415187968, 'completed': '82.97% (3_059 / 3_687)', 'remaining time': '5:35:37', 'throughput': '8285.85', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 20:46:54,274 >> {'loss': 0.4292, 'grad_norm': 20.04188346862793, 'learning_rate': 1.6932736893466536e-07, 'epoch': 0.1790073230268511, 'num_input_tokens_seen': 6417285120, 'completed': '82.99% (3_060 / 3_687)', 'remaining time': '5:35:04', 'throughput': '8372.78', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 20:47:27,124 >> {'loss': 0.3455, 'grad_norm': 15.362913131713867, 'learning_rate': 1.691122434494818e-07, 'epoch': 0.17927854624355846, 'num_input_tokens_seen': 6419382272, 'completed': '83.02% (3_061 / 3_687)', 'remaining time': '5:34:33', 'throughput': '7980.16', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 20:47:56,352 >> {'loss': 0.5172, 'grad_norm': 17.9241943359375, 'learning_rate': 1.688974244882709e-07, 'epoch': 0.1795497694602658, 'num_input_tokens_seen': 6421479424, 'completed': '83.05% (3_062 / 3_687)', 'remaining time': '5:33:58', 'throughput': '8968.79', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 20:48:28,825 >> {'loss': 0.4723, 'grad_norm': 16.651180267333984, 'learning_rate': 1.686829122239108e-07, 'epoch': 0.17982099267697316, 'num_input_tokens_seen': 6423576576, 'completed': '83.08% (3_063 / 3_687)', 'remaining time': '5:33:26', 'throughput': '8072.68', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 20:48:58,210 >> {'loss': 0.5919, 'grad_norm': 21.878660202026367, 'learning_rate': 1.684687068290326e-07, 'epoch': 0.1800922158936805, 'num_input_tokens_seen': 6425673728, 'completed': '83.10% (3_064 / 3_687)', 'remaining time': '5:32:52', 'throughput': '8921.06', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 20:49:27,462 >> {'loss': 0.3433, 'grad_norm': 16.2543888092041, 'learning_rate': 1.682548084760208e-07, 'epoch': 0.18036343911038785, 'num_input_tokens_seen': 6427770880, 'completed': '83.13% (3_065 / 3_687)', 'remaining time': '5:32:17', 'throughput': '8961.63', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 20:50:00,821 >> {'loss': 0.3266, 'grad_norm': 14.593696594238281, 'learning_rate': 1.6804121733701277e-07, 'epoch': 0.18063466232709519, 'num_input_tokens_seen': 6429868032, 'completed': '83.16% (3_066 / 3_687)', 'remaining time': '5:31:46', 'throughput': '7858.15', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 20:50:28,267 >> {'loss': 1.0168, 'grad_norm': 30.589252471923828, 'learning_rate': 1.678279335838983e-07, 'epoch': 0.18090588554380255, 'num_input_tokens_seen': 6431965184, 'completed': '83.18% (3_067 / 3_687)', 'remaining time': '5:31:10', 'throughput': '9551.24', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 20:50:59,994 >> {'loss': 0.2938, 'grad_norm': 16.187679290771484, 'learning_rate': 1.676149573883202e-07, 'epoch': 0.1811771087605099, 'num_input_tokens_seen': 6434062336, 'completed': '83.21% (3_068 / 3_687)', 'remaining time': '5:30:37', 'throughput': '8262.69', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 20:51:29,936 >> {'loss': 0.4911, 'grad_norm': 20.22064208984375, 'learning_rate': 1.674022889216737e-07, 'epoch': 0.18144833197721724, 'num_input_tokens_seen': 6436159488, 'completed': '83.24% (3_069 / 3_687)', 'remaining time': '5:30:03', 'throughput': '8755.03', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 20:52:00,835 >> {'loss': 0.3566, 'grad_norm': 15.452803611755371, 'learning_rate': 1.67189928355106e-07, 'epoch': 0.1817195551939246, 'num_input_tokens_seen': 6438256640, 'completed': '83.27% (3_070 / 3_687)', 'remaining time': '5:29:30', 'throughput': '8483.81', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 20:52:27,848 >> {'loss': 0.4931, 'grad_norm': 21.338388442993164, 'learning_rate': 1.66977875859517e-07, 'epoch': 0.18199077841063194, 'num_input_tokens_seen': 6440353792, 'completed': '83.29% (3_071 / 3_687)', 'remaining time': '5:28:54', 'throughput': '9704.49', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 20:52:57,315 >> {'loss': 0.3819, 'grad_norm': 15.379291534423828, 'learning_rate': 1.6676613160555846e-07, 'epoch': 0.1822620016273393, 'num_input_tokens_seen': 6442450944, 'completed': '83.32% (3_072 / 3_687)', 'remaining time': '5:28:19', 'throughput': '8896.25', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 20:53:25,427 >> {'loss': 0.5989, 'grad_norm': 18.543752670288086, 'learning_rate': 1.665546957636339e-07, 'epoch': 0.18253322484404666, 'num_input_tokens_seen': 6444548096, 'completed': '83.35% (3_073 / 3_687)', 'remaining time': '5:27:44', 'throughput': '9324.98', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 20:53:56,277 >> {'loss': 0.3388, 'grad_norm': 13.839360237121582, 'learning_rate': 1.6634356850389926e-07, 'epoch': 0.182804448060754, 'num_input_tokens_seen': 6446645248, 'completed': '83.37% (3_074 / 3_687)', 'remaining time': '5:27:11', 'throughput': '8497.21', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 20:54:27,434 >> {'loss': 0.4291, 'grad_norm': 14.323488235473633, 'learning_rate': 1.6613274999626134e-07, 'epoch': 0.18307567127746135, 'num_input_tokens_seen': 6448742400, 'completed': '83.40% (3_075 / 3_687)', 'remaining time': '5:26:38', 'throughput': '8413.61', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 20:54:56,018 >> {'loss': 0.5632, 'grad_norm': 19.329299926757812, 'learning_rate': 1.65922240410379e-07, 'epoch': 0.1833468944941687, 'num_input_tokens_seen': 6450839552, 'completed': '83.43% (3_076 / 3_687)', 'remaining time': '5:26:03', 'throughput': '9170.94', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 20:55:30,364 >> {'loss': 0.3609, 'grad_norm': 14.438604354858398, 'learning_rate': 1.6571203991566234e-07, 'epoch': 0.18361811771087605, 'num_input_tokens_seen': 6452936704, 'completed': '83.46% (3_077 / 3_687)', 'remaining time': '5:25:33', 'throughput': '7632.49', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 20:56:00,668 >> {'loss': 0.2715, 'grad_norm': 16.972644805908203, 'learning_rate': 1.6550214868127276e-07, 'epoch': 0.1838893409275834, 'num_input_tokens_seen': 6455033856, 'completed': '83.48% (3_078 / 3_687)', 'remaining time': '5:24:59', 'throughput': '8650.52', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 20:56:33,428 >> {'loss': 0.5057, 'grad_norm': 19.990781784057617, 'learning_rate': 1.6529256687612264e-07, 'epoch': 0.18416056414429074, 'num_input_tokens_seen': 6457131008, 'completed': '83.51% (3_079 / 3_687)', 'remaining time': '5:24:28', 'throughput': '8001.95', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 20:57:05,885 >> {'loss': 0.32, 'grad_norm': 12.420730590820312, 'learning_rate': 1.6508329466887548e-07, 'epoch': 0.1844317873609981, 'num_input_tokens_seen': 6459228160, 'completed': '83.54% (3_080 / 3_687)', 'remaining time': '5:23:56', 'throughput': '8076.59', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 20:57:36,242 >> {'loss': 0.531, 'grad_norm': 18.54897689819336, 'learning_rate': 1.6487433222794566e-07, 'epoch': 0.18470301057770544, 'num_input_tokens_seen': 6461325312, 'completed': '83.56% (3_081 / 3_687)', 'remaining time': '5:23:23', 'throughput': '8635.39', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 20:58:08,086 >> {'loss': 0.2632, 'grad_norm': 13.507473945617676, 'learning_rate': 1.6466567972149806e-07, 'epoch': 0.1849742337944128, 'num_input_tokens_seen': 6463422464, 'completed': '83.59% (3_082 / 3_687)', 'remaining time': '5:22:50', 'throughput': '8232.32', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 20:58:39,022 >> {'loss': 0.3308, 'grad_norm': 13.425512313842773, 'learning_rate': 1.6445733731744842e-07, 'epoch': 0.18524545701112016, 'num_input_tokens_seen': 6465519616, 'completed': '83.62% (3_083 / 3_687)', 'remaining time': '5:22:17', 'throughput': '8473.68', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 20:59:13,507 >> {'loss': 0.3104, 'grad_norm': 14.376126289367676, 'learning_rate': 1.642493051834627e-07, 'epoch': 0.1855166802278275, 'num_input_tokens_seen': 6467616768, 'completed': '83.65% (3_084 / 3_687)', 'remaining time': '5:21:48', 'throughput': '7601.57', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 20:59:42,931 >> {'loss': 0.3902, 'grad_norm': 13.848085403442383, 'learning_rate': 1.6404158348695729e-07, 'epoch': 0.18578790344453486, 'num_input_tokens_seen': 6469713920, 'completed': '83.67% (3_085 / 3_687)', 'remaining time': '5:21:13', 'throughput': '8909.23', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 21:00:11,886 >> {'loss': 0.5748, 'grad_norm': 17.45004653930664, 'learning_rate': 1.6383417239509878e-07, 'epoch': 0.1860591266612422, 'num_input_tokens_seen': 6471811072, 'completed': '83.70% (3_086 / 3_687)', 'remaining time': '5:20:39', 'throughput': '9053.71', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 21:00:41,329 >> {'loss': 0.6891, 'grad_norm': 19.52586555480957, 'learning_rate': 1.6362707207480347e-07, 'epoch': 0.18633034987794955, 'num_input_tokens_seen': 6473908224, 'completed': '83.73% (3_087 / 3_687)', 'remaining time': '5:20:04', 'throughput': '8904.27', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 21:01:12,553 >> {'loss': 0.6888, 'grad_norm': 24.07430076599121, 'learning_rate': 1.6342028269273802e-07, 'epoch': 0.18660157309465691, 'num_input_tokens_seen': 6476005376, 'completed': '83.75% (3_088 / 3_687)', 'remaining time': '5:19:32', 'throughput': '8394.75', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 21:01:43,411 >> {'loss': 0.6213, 'grad_norm': 19.478736877441406, 'learning_rate': 1.6321380441531852e-07, 'epoch': 0.18687279631136425, 'num_input_tokens_seen': 6478102528, 'completed': '83.78% (3_089 / 3_687)', 'remaining time': '5:18:59', 'throughput': '8495.12', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 21:02:15,067 >> {'loss': 0.3439, 'grad_norm': 13.465045928955078, 'learning_rate': 1.63007637408711e-07, 'epoch': 0.1871440195280716, 'num_input_tokens_seen': 6480199680, 'completed': '83.81% (3_090 / 3_687)', 'remaining time': '5:18:26', 'throughput': '8281.19', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 21:02:44,175 >> {'loss': 0.4287, 'grad_norm': 16.045684814453125, 'learning_rate': 1.6280178183883081e-07, 'epoch': 0.18741524274477894, 'num_input_tokens_seen': 6482296832, 'completed': '83.84% (3_091 / 3_687)', 'remaining time': '5:17:52', 'throughput': '9005.88', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 21:03:13,848 >> {'loss': 0.4006, 'grad_norm': 18.508325576782227, 'learning_rate': 1.6259623787134258e-07, 'epoch': 0.1876864659614863, 'num_input_tokens_seen': 6484393984, 'completed': '83.86% (3_092 / 3_687)', 'remaining time': '5:17:18', 'throughput': '8834.23', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 21:03:44,061 >> {'loss': 0.392, 'grad_norm': 16.57149314880371, 'learning_rate': 1.6239100567166026e-07, 'epoch': 0.18795768917819367, 'num_input_tokens_seen': 6486491136, 'completed': '83.89% (3_093 / 3_687)', 'remaining time': '5:16:44', 'throughput': '8676.62', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 21:04:14,989 >> {'loss': 0.3385, 'grad_norm': 13.103677749633789, 'learning_rate': 1.6218608540494693e-07, 'epoch': 0.188228912394901, 'num_input_tokens_seen': 6488588288, 'completed': '83.92% (3_094 / 3_687)', 'remaining time': '5:16:11', 'throughput': '8475.86', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 21:04:44,447 >> {'loss': 0.3766, 'grad_norm': 17.10627555847168, 'learning_rate': 1.619814772361147e-07, 'epoch': 0.18850013561160836, 'num_input_tokens_seen': 6490685440, 'completed': '83.94% (3_095 / 3_687)', 'remaining time': '5:15:37', 'throughput': '8898.98', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 21:05:16,183 >> {'loss': 0.5857, 'grad_norm': 18.120052337646484, 'learning_rate': 1.6177718132982441e-07, 'epoch': 0.1887713588283157, 'num_input_tokens_seen': 6492782592, 'completed': '83.97% (3_096 / 3_687)', 'remaining time': '5:15:05', 'throughput': '8260.20', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 21:05:46,533 >> {'loss': 0.5742, 'grad_norm': 19.000185012817383, 'learning_rate': 1.6157319785048555e-07, 'epoch': 0.18904258204502306, 'num_input_tokens_seen': 6494879744, 'completed': '84.00% (3_097 / 3_687)', 'remaining time': '5:14:32', 'throughput': '8637.33', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 21:06:20,099 >> {'loss': 0.3558, 'grad_norm': 18.840848922729492, 'learning_rate': 1.6136952696225634e-07, 'epoch': 0.18931380526173042, 'num_input_tokens_seen': 6496976896, 'completed': '84.02% (3_098 / 3_687)', 'remaining time': '5:14:01', 'throughput': '7809.73', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 21:06:56,160 >> {'loss': 0.5141, 'grad_norm': 19.136682510375977, 'learning_rate': 1.6116616882904332e-07, 'epoch': 0.18958502847843775, 'num_input_tokens_seen': 6499074048, 'completed': '84.05% (3_099 / 3_687)', 'remaining time': '5:13:32', 'throughput': '7269.58', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 21:07:31,817 >> {'loss': 0.434, 'grad_norm': 17.346790313720703, 'learning_rate': 1.6096312361450142e-07, 'epoch': 0.1898562516951451, 'num_input_tokens_seen': 6501171200, 'completed': '84.08% (3_100 / 3_687)', 'remaining time': '5:13:04', 'throughput': '7351.78', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 21:08:01,295 >> {'loss': 0.4427, 'grad_norm': 15.207001686096191, 'learning_rate': 1.6076039148203373e-07, 'epoch': 0.19012747491185245, 'num_input_tokens_seen': 6503268352, 'completed': '84.11% (3_101 / 3_687)', 'remaining time': '5:12:29', 'throughput': '8892.78', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 21:08:30,704 >> {'loss': 0.253, 'grad_norm': 10.525275230407715, 'learning_rate': 1.6055797259479125e-07, 'epoch': 0.1903986981285598, 'num_input_tokens_seen': 6505365504, 'completed': '84.13% (3_102 / 3_687)', 'remaining time': '5:11:55', 'throughput': '8913.87', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 21:09:04,044 >> {'loss': 0.6012, 'grad_norm': 27.057727813720703, 'learning_rate': 1.6035586711567318e-07, 'epoch': 0.19066992134526717, 'num_input_tokens_seen': 6507462656, 'completed': '84.16% (3_103 / 3_687)', 'remaining time': '5:11:24', 'throughput': '7862.73', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 21:09:32,001 >> {'loss': 0.673, 'grad_norm': 29.3339786529541, 'learning_rate': 1.601540752073261e-07, 'epoch': 0.1909411445619745, 'num_input_tokens_seen': 6509559808, 'completed': '84.19% (3_104 / 3_687)', 'remaining time': '5:10:49', 'throughput': '9376.76', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 21:10:05,277 >> {'loss': 0.2948, 'grad_norm': 12.424835205078125, 'learning_rate': 1.5995259703214445e-07, 'epoch': 0.19121236777868186, 'num_input_tokens_seen': 6511656960, 'completed': '84.21% (3_105 / 3_687)', 'remaining time': '5:10:18', 'throughput': '7877.80', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 21:10:32,928 >> {'loss': 0.3906, 'grad_norm': 14.519540786743164, 'learning_rate': 1.5975143275227039e-07, 'epoch': 0.1914835909953892, 'num_input_tokens_seen': 6513754112, 'completed': '84.24% (3_106 / 3_687)', 'remaining time': '5:09:43', 'throughput': '9480.42', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 21:11:04,446 >> {'loss': 0.4539, 'grad_norm': 18.084651947021484, 'learning_rate': 1.5955058252959318e-07, 'epoch': 0.19175481421209656, 'num_input_tokens_seen': 6515851264, 'completed': '84.27% (3_107 / 3_687)', 'remaining time': '5:09:10', 'throughput': '8317.29', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 21:11:32,669 >> {'loss': 0.5936, 'grad_norm': 23.673160552978516, 'learning_rate': 1.5935004652574947e-07, 'epoch': 0.1920260374288039, 'num_input_tokens_seen': 6517948416, 'completed': '84.30% (3_108 / 3_687)', 'remaining time': '5:08:35', 'throughput': '9288.41', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 21:12:02,814 >> {'loss': 0.2686, 'grad_norm': 13.00130844116211, 'learning_rate': 1.591498249021231e-07, 'epoch': 0.19229726064551125, 'num_input_tokens_seen': 6520045568, 'completed': '84.32% (3_109 / 3_687)', 'remaining time': '5:08:02', 'throughput': '8696.02', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 21:12:35,445 >> {'loss': 0.6705, 'grad_norm': 21.13102912902832, 'learning_rate': 1.5894991781984456e-07, 'epoch': 0.19256848386221861, 'num_input_tokens_seen': 6522142720, 'completed': '84.35% (3_110 / 3_687)', 'remaining time': '5:07:30', 'throughput': '8033.49', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 21:13:05,713 >> {'loss': 0.1552, 'grad_norm': 10.354960441589355, 'learning_rate': 1.587503254397916e-07, 'epoch': 0.19283970707892595, 'num_input_tokens_seen': 6524239872, 'completed': '84.38% (3_111 / 3_687)', 'remaining time': '5:06:57', 'throughput': '8660.94', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 21:13:35,292 >> {'loss': 0.2692, 'grad_norm': 16.40614128112793, 'learning_rate': 1.585510479225886e-07, 'epoch': 0.1931109302956333, 'num_input_tokens_seen': 6526337024, 'completed': '84.40% (3_112 / 3_687)', 'remaining time': '5:06:23', 'throughput': '8862.51', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 21:14:04,560 >> {'loss': 0.3284, 'grad_norm': 14.614860534667969, 'learning_rate': 1.5835208542860648e-07, 'epoch': 0.19338215351234064, 'num_input_tokens_seen': 6528434176, 'completed': '84.43% (3_113 / 3_687)', 'remaining time': '5:05:49', 'throughput': '8956.54', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 21:14:37,002 >> {'loss': 0.3218, 'grad_norm': 11.804844856262207, 'learning_rate': 1.581534381179627e-07, 'epoch': 0.193653376729048, 'num_input_tokens_seen': 6530531328, 'completed': '84.46% (3_114 / 3_687)', 'remaining time': '5:05:17', 'throughput': '8080.53', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 21:15:06,927 >> {'loss': 0.5768, 'grad_norm': 19.765899658203125, 'learning_rate': 1.5795510615052104e-07, 'epoch': 0.19392459994575537, 'num_input_tokens_seen': 6532628480, 'completed': '84.49% (3_115 / 3_687)', 'remaining time': '5:04:44', 'throughput': '8759.86', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 21:15:39,626 >> {'loss': 0.4025, 'grad_norm': 18.440826416015625, 'learning_rate': 1.5775708968589155e-07, 'epoch': 0.1941958231624627, 'num_input_tokens_seen': 6534725632, 'completed': '84.51% (3_116 / 3_687)', 'remaining time': '5:04:12', 'throughput': '8016.96', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 21:16:09,896 >> {'loss': 0.4297, 'grad_norm': 27.12670135498047, 'learning_rate': 1.575593888834303e-07, 'epoch': 0.19446704637917006, 'num_input_tokens_seen': 6536822784, 'completed': '84.54% (3_117 / 3_687)', 'remaining time': '5:03:39', 'throughput': '8660.12', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 21:16:38,024 >> {'loss': 0.7779, 'grad_norm': 22.037416458129883, 'learning_rate': 1.5736200390223942e-07, 'epoch': 0.1947382695958774, 'num_input_tokens_seen': 6538919936, 'completed': '84.57% (3_118 / 3_687)', 'remaining time': '5:03:04', 'throughput': '9319.74', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 21:17:07,006 >> {'loss': 0.3269, 'grad_norm': 15.226699829101562, 'learning_rate': 1.5716493490116684e-07, 'epoch': 0.19500949281258476, 'num_input_tokens_seen': 6541017088, 'completed': '84.59% (3_119 / 3_687)', 'remaining time': '5:02:30', 'throughput': '9045.14', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 21:17:37,504 >> {'loss': 0.4225, 'grad_norm': 19.05426788330078, 'learning_rate': 1.5696818203880624e-07, 'epoch': 0.19528071602929212, 'num_input_tokens_seen': 6543114240, 'completed': '84.62% (3_120 / 3_687)', 'remaining time': '5:01:57', 'throughput': '8595.27', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 21:18:06,060 >> {'loss': 0.561, 'grad_norm': 15.81242847442627, 'learning_rate': 1.5677174547349655e-07, 'epoch': 0.19555193924599945, 'num_input_tokens_seen': 6545211392, 'completed': '84.65% (3_121 / 3_687)', 'remaining time': '5:01:22', 'throughput': '9180.00', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 21:18:35,783 >> {'loss': 0.5197, 'grad_norm': 24.574951171875, 'learning_rate': 1.565756253633228e-07, 'epoch': 0.1958231624627068, 'num_input_tokens_seen': 6547308544, 'completed': '84.68% (3_122 / 3_687)', 'remaining time': '5:00:48', 'throughput': '8819.61', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 21:19:07,112 >> {'loss': 0.4524, 'grad_norm': 21.462337493896484, 'learning_rate': 1.5637982186611481e-07, 'epoch': 0.19609438567941415, 'num_input_tokens_seen': 6549405696, 'completed': '84.70% (3_123 / 3_687)', 'remaining time': '5:00:16', 'throughput': '8367.44', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 21:19:36,680 >> {'loss': 0.2997, 'grad_norm': 18.190275192260742, 'learning_rate': 1.5618433513944787e-07, 'epoch': 0.1963656088961215, 'num_input_tokens_seen': 6551502848, 'completed': '84.73% (3_124 / 3_687)', 'remaining time': '4:59:42', 'throughput': '8865.82', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 21:20:07,741 >> {'loss': 0.5017, 'grad_norm': 18.96411895751953, 'learning_rate': 1.5598916534064216e-07, 'epoch': 0.19663683211282887, 'num_input_tokens_seen': 6553600000, 'completed': '84.76% (3_125 / 3_687)', 'remaining time': '4:59:09', 'throughput': '8439.73', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 21:20:39,566 >> {'loss': 0.4162, 'grad_norm': 16.543214797973633, 'learning_rate': 1.557943126267631e-07, 'epoch': 0.1969080553295362, 'num_input_tokens_seen': 6555697152, 'completed': '84.78% (3_126 / 3_687)', 'remaining time': '4:58:37', 'throughput': '8236.95', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 21:21:08,790 >> {'loss': 0.287, 'grad_norm': 13.697303771972656, 'learning_rate': 1.5559977715462044e-07, 'epoch': 0.19717927854624356, 'num_input_tokens_seen': 6557794304, 'completed': '84.81% (3_127 / 3_687)', 'remaining time': '4:58:03', 'throughput': '8970.21', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 21:21:40,339 >> {'loss': 0.6185, 'grad_norm': 19.32088279724121, 'learning_rate': 1.5540555908076914e-07, 'epoch': 0.1974505017629509, 'num_input_tokens_seen': 6559891456, 'completed': '84.84% (3_128 / 3_687)', 'remaining time': '4:57:31', 'throughput': '8310.21', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 21:22:12,235 >> {'loss': 0.4217, 'grad_norm': 21.070152282714844, 'learning_rate': 1.5521165856150841e-07, 'epoch': 0.19772172497965826, 'num_input_tokens_seen': 6561988608, 'completed': '84.87% (3_129 / 3_687)', 'remaining time': '4:56:59', 'throughput': '8217.75', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 21:22:43,007 >> {'loss': 0.5169, 'grad_norm': 27.660669326782227, 'learning_rate': 1.55018075752882e-07, 'epoch': 0.19799294819636562, 'num_input_tokens_seen': 6564085760, 'completed': '84.89% (3_130 / 3_687)', 'remaining time': '4:56:26', 'throughput': '8518.82', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 21:23:12,218 >> {'loss': 0.4193, 'grad_norm': 17.022554397583008, 'learning_rate': 1.5482481081067825e-07, 'epoch': 0.19826417141307295, 'num_input_tokens_seen': 6566182912, 'completed': '84.92% (3_131 / 3_687)', 'remaining time': '4:55:52', 'throughput': '8974.11', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 21:23:41,642 >> {'loss': 0.4522, 'grad_norm': 18.43861198425293, 'learning_rate': 1.546318638904291e-07, 'epoch': 0.19853539462978032, 'num_input_tokens_seen': 6568280064, 'completed': '84.95% (3_132 / 3_687)', 'remaining time': '4:55:18', 'throughput': '8909.18', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 21:24:14,799 >> {'loss': 0.2211, 'grad_norm': 15.052275657653809, 'learning_rate': 1.5443923514741103e-07, 'epoch': 0.19880661784648765, 'num_input_tokens_seen': 6570377216, 'completed': '84.97% (3_133 / 3_687)', 'remaining time': '4:54:47', 'throughput': '7906.15', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 21:24:47,465 >> {'loss': 0.3405, 'grad_norm': 21.32815933227539, 'learning_rate': 1.5424692473664429e-07, 'epoch': 0.199077841063195, 'num_input_tokens_seen': 6572474368, 'completed': '85.00% (3_134 / 3_687)', 'remaining time': '4:54:16', 'throughput': '8025.08', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 21:25:17,069 >> {'loss': 0.6306, 'grad_norm': 20.877336502075195, 'learning_rate': 1.5405493281289302e-07, 'epoch': 0.19934906427990237, 'num_input_tokens_seen': 6574571520, 'completed': '85.03% (3_135 / 3_687)', 'remaining time': '4:53:42', 'throughput': '8855.03', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 21:25:48,191 >> {'loss': 0.6523, 'grad_norm': 21.885692596435547, 'learning_rate': 1.5386325953066494e-07, 'epoch': 0.1996202874966097, 'num_input_tokens_seen': 6576668672, 'completed': '85.06% (3_136 / 3_687)', 'remaining time': '4:53:10', 'throughput': '8423.15', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 21:26:19,131 >> {'loss': 0.3159, 'grad_norm': 13.137401580810547, 'learning_rate': 1.5367190504421157e-07, 'epoch': 0.19989151071331707, 'num_input_tokens_seen': 6578765824, 'completed': '85.08% (3_137 / 3_687)', 'remaining time': '4:52:37', 'throughput': '8472.58', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 21:26:49,849 >> {'loss': 0.3431, 'grad_norm': 15.956317901611328, 'learning_rate': 1.5348086950752753e-07, 'epoch': 0.2001627339300244, 'num_input_tokens_seen': 6580862976, 'completed': '85.11% (3_138 / 3_687)', 'remaining time': '4:52:04', 'throughput': '8533.93', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 21:27:20,374 >> {'loss': 0.4469, 'grad_norm': 16.24814796447754, 'learning_rate': 1.532901530743511e-07, 'epoch': 0.20043395714673176, 'num_input_tokens_seen': 6582960128, 'completed': '85.14% (3_139 / 3_687)', 'remaining time': '4:51:31', 'throughput': '8587.83', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 21:27:50,529 >> {'loss': 0.4047, 'grad_norm': 13.850639343261719, 'learning_rate': 1.530997558981635e-07, 'epoch': 0.20070518036343912, 'num_input_tokens_seen': 6585057280, 'completed': '85.16% (3_140 / 3_687)', 'remaining time': '4:50:58', 'throughput': '8693.13', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 21:28:23,013 >> {'loss': 0.7333, 'grad_norm': 21.258054733276367, 'learning_rate': 1.5290967813218919e-07, 'epoch': 0.20097640358014646, 'num_input_tokens_seen': 6587154432, 'completed': '85.19% (3_141 / 3_687)', 'remaining time': '4:50:27', 'throughput': '8069.98', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 21:28:58,984 >> {'loss': 0.3701, 'grad_norm': 24.493776321411133, 'learning_rate': 1.527199199293955e-07, 'epoch': 0.20124762679685382, 'num_input_tokens_seen': 6589251584, 'completed': '85.22% (3_142 / 3_687)', 'remaining time': '4:49:58', 'throughput': '7287.59', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 21:29:31,387 >> {'loss': 0.7751, 'grad_norm': 25.23069190979004, 'learning_rate': 1.525304814424927e-07, 'epoch': 0.20151885001356115, 'num_input_tokens_seen': 6591348736, 'completed': '85.25% (3_143 / 3_687)', 'remaining time': '4:49:26', 'throughput': '8090.27', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 21:30:03,194 >> {'loss': 0.3786, 'grad_norm': 16.206256866455078, 'learning_rate': 1.5234136282393356e-07, 'epoch': 0.2017900732302685, 'num_input_tokens_seen': 6593445888, 'completed': '85.27% (3_144 / 3_687)', 'remaining time': '4:48:54', 'throughput': '8241.70', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 21:30:32,039 >> {'loss': 0.5419, 'grad_norm': 18.445281982421875, 'learning_rate': 1.5215256422591357e-07, 'epoch': 0.20206129644697587, 'num_input_tokens_seen': 6595543040, 'completed': '85.30% (3_145 / 3_687)', 'remaining time': '4:48:20', 'throughput': '9087.80', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 21:31:04,195 >> {'loss': 0.225, 'grad_norm': 13.487834930419922, 'learning_rate': 1.5196408580037058e-07, 'epoch': 0.2023325196636832, 'num_input_tokens_seen': 6597640192, 'completed': '85.33% (3_146 / 3_687)', 'remaining time': '4:47:48', 'throughput': '8152.29', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 21:31:34,091 >> {'loss': 0.2773, 'grad_norm': 18.042003631591797, 'learning_rate': 1.51775927698985e-07, 'epoch': 0.20260374288039057, 'num_input_tokens_seen': 6599737344, 'completed': '85.35% (3_147 / 3_687)', 'remaining time': '4:47:15', 'throughput': '8768.60', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 21:32:04,517 >> {'loss': 0.3948, 'grad_norm': 14.105890274047852, 'learning_rate': 1.5158809007317926e-07, 'epoch': 0.2028749660970979, 'num_input_tokens_seen': 6601834496, 'completed': '85.38% (3_148 / 3_687)', 'remaining time': '4:46:42', 'throughput': '8615.68', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 21:32:33,763 >> {'loss': 0.7842, 'grad_norm': 22.028852462768555, 'learning_rate': 1.514005730741181e-07, 'epoch': 0.20314618931380526, 'num_input_tokens_seen': 6603931648, 'completed': '85.41% (3_149 / 3_687)', 'remaining time': '4:46:08', 'throughput': '8963.43', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 21:33:03,119 >> {'loss': 0.4541, 'grad_norm': 18.55408477783203, 'learning_rate': 1.512133768527077e-07, 'epoch': 0.20341741253051263, 'num_input_tokens_seen': 6606028800, 'completed': '85.44% (3_150 / 3_687)', 'remaining time': '4:45:34', 'throughput': '8929.93', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 21:33:32,847 >> {'loss': 0.3687, 'grad_norm': 15.917207717895508, 'learning_rate': 1.510265015595967e-07, 'epoch': 0.20368863574721996, 'num_input_tokens_seen': 6608125952, 'completed': '85.46% (3_151 / 3_687)', 'remaining time': '4:45:01', 'throughput': '8818.16', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 21:34:02,276 >> {'loss': 0.6256, 'grad_norm': 25.865598678588867, 'learning_rate': 1.5083994734517518e-07, 'epoch': 0.20395985896392732, 'num_input_tokens_seen': 6610223104, 'completed': '85.49% (3_152 / 3_687)', 'remaining time': '4:44:27', 'throughput': '8907.68', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 21:34:31,254 >> {'loss': 0.4382, 'grad_norm': 17.131990432739258, 'learning_rate': 1.5065371435957484e-07, 'epoch': 0.20423108218063465, 'num_input_tokens_seen': 6612320256, 'completed': '85.52% (3_153 / 3_687)', 'remaining time': '4:43:53', 'throughput': '9046.07', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 21:35:02,496 >> {'loss': 0.7498, 'grad_norm': 24.471385955810547, 'learning_rate': 1.5046780275266897e-07, 'epoch': 0.20450230539734202, 'num_input_tokens_seen': 6614417408, 'completed': '85.54% (3_154 / 3_687)', 'remaining time': '4:43:21', 'throughput': '8390.86', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 21:35:35,883 >> {'loss': 0.4527, 'grad_norm': 17.423646926879883, 'learning_rate': 1.5028221267407207e-07, 'epoch': 0.20477352861404935, 'num_input_tokens_seen': 6616514560, 'completed': '85.57% (3_155 / 3_687)', 'remaining time': '4:42:50', 'throughput': '7852.13', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 21:36:05,624 >> {'loss': 0.2501, 'grad_norm': 13.580658912658691, 'learning_rate': 1.5009694427314007e-07, 'epoch': 0.2050447518307567, 'num_input_tokens_seen': 6618611712, 'completed': '85.60% (3_156 / 3_687)', 'remaining time': '4:42:17', 'throughput': '8813.55', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 21:36:38,590 >> {'loss': 0.33, 'grad_norm': 16.855371475219727, 'learning_rate': 1.4991199769896983e-07, 'epoch': 0.20531597504746407, 'num_input_tokens_seen': 6620708864, 'completed': '85.63% (3_157 / 3_687)', 'remaining time': '4:41:45', 'throughput': '7952.03', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 21:37:09,248 >> {'loss': 0.3885, 'grad_norm': 14.530333518981934, 'learning_rate': 1.497273731003994e-07, 'epoch': 0.2055871982641714, 'num_input_tokens_seen': 6622806016, 'completed': '85.65% (3_158 / 3_687)', 'remaining time': '4:41:13', 'throughput': '8550.67', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 21:37:40,136 >> {'loss': 0.3511, 'grad_norm': 13.189663887023926, 'learning_rate': 1.4954307062600758e-07, 'epoch': 0.20585842148087877, 'num_input_tokens_seen': 6624903168, 'completed': '85.68% (3_159 / 3_687)', 'remaining time': '4:40:40', 'throughput': '8486.80', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 21:38:10,012 >> {'loss': 0.6974, 'grad_norm': 21.233369827270508, 'learning_rate': 1.4935909042411412e-07, 'epoch': 0.2061296446975861, 'num_input_tokens_seen': 6627000320, 'completed': '85.71% (3_160 / 3_687)', 'remaining time': '4:40:07', 'throughput': '8774.44', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 21:38:38,842 >> {'loss': 0.498, 'grad_norm': 19.892419815063477, 'learning_rate': 1.4917543264277901e-07, 'epoch': 0.20640086791429346, 'num_input_tokens_seen': 6629097472, 'completed': '85.73% (3_161 / 3_687)', 'remaining time': '4:39:33', 'throughput': '9092.89', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 21:39:08,510 >> {'loss': 0.4197, 'grad_norm': 14.9336519241333, 'learning_rate': 1.4899209742980317e-07, 'epoch': 0.20667209113100082, 'num_input_tokens_seen': 6631194624, 'completed': '85.76% (3_162 / 3_687)', 'remaining time': '4:38:59', 'throughput': '8835.87', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 21:39:38,679 >> {'loss': 0.5148, 'grad_norm': 19.203628540039062, 'learning_rate': 1.488090849327279e-07, 'epoch': 0.20694331434770816, 'num_input_tokens_seen': 6633291776, 'completed': '85.79% (3_163 / 3_687)', 'remaining time': '4:38:26', 'throughput': '8689.10', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 21:40:09,128 >> {'loss': 0.3568, 'grad_norm': 14.907687187194824, 'learning_rate': 1.4862639529883463e-07, 'epoch': 0.20721453756441552, 'num_input_tokens_seen': 6635388928, 'completed': '85.82% (3_164 / 3_687)', 'remaining time': '4:37:53', 'throughput': '8609.18', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 21:40:39,554 >> {'loss': 0.3888, 'grad_norm': 16.64046287536621, 'learning_rate': 1.4844402867514503e-07, 'epoch': 0.20748576078112285, 'num_input_tokens_seen': 6637486080, 'completed': '85.84% (3_165 / 3_687)', 'remaining time': '4:37:21', 'throughput': '8615.78', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 21:41:13,119 >> {'loss': 0.2793, 'grad_norm': 12.00853157043457, 'learning_rate': 1.4826198520842093e-07, 'epoch': 0.2077569839978302, 'num_input_tokens_seen': 6639583232, 'completed': '85.87% (3_166 / 3_687)', 'remaining time': '4:36:50', 'throughput': '7810.24', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 21:41:44,109 >> {'loss': 0.4134, 'grad_norm': 15.106518745422363, 'learning_rate': 1.4808026504516374e-07, 'epoch': 0.20802820721453757, 'num_input_tokens_seen': 6641680384, 'completed': '85.90% (3_167 / 3_687)', 'remaining time': '4:36:17', 'throughput': '8458.87', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 21:42:13,338 >> {'loss': 0.5532, 'grad_norm': 18.982995986938477, 'learning_rate': 1.4789886833161506e-07, 'epoch': 0.2082994304312449, 'num_input_tokens_seen': 6643777536, 'completed': '85.92% (3_168 / 3_687)', 'remaining time': '4:35:44', 'throughput': '8968.79', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 21:42:45,922 >> {'loss': 0.3146, 'grad_norm': 12.5213041305542, 'learning_rate': 1.477177952137561e-07, 'epoch': 0.20857065364795227, 'num_input_tokens_seen': 6645874688, 'completed': '85.95% (3_169 / 3_687)', 'remaining time': '4:35:12', 'throughput': '8045.06', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 21:43:20,034 >> {'loss': 0.5158, 'grad_norm': 18.02132797241211, 'learning_rate': 1.4753704583730754e-07, 'epoch': 0.2088418768646596, 'num_input_tokens_seen': 6647971840, 'completed': '85.98% (3_170 / 3_687)', 'remaining time': '4:34:42', 'throughput': '7684.82', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 21:43:52,122 >> {'loss': 0.3841, 'grad_norm': 16.98950958251953, 'learning_rate': 1.4735662034772962e-07, 'epoch': 0.20911310008136696, 'num_input_tokens_seen': 6650068992, 'completed': '86.00% (3_171 / 3_687)', 'remaining time': '4:34:10', 'throughput': '8169.60', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 21:44:22,021 >> {'loss': 0.327, 'grad_norm': 15.854743003845215, 'learning_rate': 1.47176518890222e-07, 'epoch': 0.20938432329807433, 'num_input_tokens_seen': 6652166144, 'completed': '86.03% (3_172 / 3_687)', 'remaining time': '4:33:37', 'throughput': '8767.53', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 21:44:53,648 >> {'loss': 0.3407, 'grad_norm': 14.455223083496094, 'learning_rate': 1.4699674160972337e-07, 'epoch': 0.20965554651478166, 'num_input_tokens_seen': 6654263296, 'completed': '86.06% (3_173 / 3_687)', 'remaining time': '4:33:05', 'throughput': '8288.66', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 21:45:22,674 >> {'loss': 0.2415, 'grad_norm': 12.59441089630127, 'learning_rate': 1.4681728865091165e-07, 'epoch': 0.20992676973148902, 'num_input_tokens_seen': 6656360448, 'completed': '86.09% (3_174 / 3_687)', 'remaining time': '4:32:31', 'throughput': '9031.41', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 21:45:51,101 >> {'loss': 0.529, 'grad_norm': 21.2336483001709, 'learning_rate': 1.466381601582038e-07, 'epoch': 0.21019799294819635, 'num_input_tokens_seen': 6658457600, 'completed': '86.11% (3_175 / 3_687)', 'remaining time': '4:31:57', 'throughput': '9221.59', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 21:46:21,161 >> {'loss': 0.2731, 'grad_norm': 15.553447723388672, 'learning_rate': 1.4645935627575562e-07, 'epoch': 0.21046921616490372, 'num_input_tokens_seen': 6660554752, 'completed': '86.14% (3_176 / 3_687)', 'remaining time': '4:31:24', 'throughput': '8720.72', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 21:46:48,845 >> {'loss': 0.626, 'grad_norm': 19.533782958984375, 'learning_rate': 1.462808771474617e-07, 'epoch': 0.21074043938161108, 'num_input_tokens_seen': 6662651904, 'completed': '86.17% (3_177 / 3_687)', 'remaining time': '4:30:49', 'throughput': '9469.21', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 21:47:19,602 >> {'loss': 0.2152, 'grad_norm': 11.384428024291992, 'learning_rate': 1.4610272291695503e-07, 'epoch': 0.2110116625983184, 'num_input_tokens_seen': 6664749056, 'completed': '86.19% (3_178 / 3_687)', 'remaining time': '4:30:17', 'throughput': '8522.97', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 21:47:50,938 >> {'loss': 0.3965, 'grad_norm': 17.49107551574707, 'learning_rate': 1.4592489372760764e-07, 'epoch': 0.21128288581502577, 'num_input_tokens_seen': 6666846208, 'completed': '86.22% (3_179 / 3_687)', 'remaining time': '4:29:45', 'throughput': '8365.56', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 21:48:22,962 >> {'loss': 0.3181, 'grad_norm': 14.759407043457031, 'learning_rate': 1.4574738972252953e-07, 'epoch': 0.2115541090317331, 'num_input_tokens_seen': 6668943360, 'completed': '86.25% (3_180 / 3_687)', 'remaining time': '4:29:13', 'throughput': '8185.88', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 21:48:52,024 >> {'loss': 0.4154, 'grad_norm': 15.087119102478027, 'learning_rate': 1.4557021104456923e-07, 'epoch': 0.21182533224844047, 'num_input_tokens_seen': 6671040512, 'completed': '86.28% (3_181 / 3_687)', 'remaining time': '4:28:39', 'throughput': '9020.01', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 21:49:24,807 >> {'loss': 0.3897, 'grad_norm': 17.55137062072754, 'learning_rate': 1.4539335783631346e-07, 'epoch': 0.21209655546514783, 'num_input_tokens_seen': 6673137664, 'completed': '86.30% (3_182 / 3_687)', 'remaining time': '4:28:08', 'throughput': '7996.34', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 21:49:56,738 >> {'loss': 0.4131, 'grad_norm': 14.304892539978027, 'learning_rate': 1.4521683024008687e-07, 'epoch': 0.21236777868185516, 'num_input_tokens_seen': 6675234816, 'completed': '86.33% (3_183 / 3_687)', 'remaining time': '4:27:36', 'throughput': '8209.92', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 21:50:27,463 >> {'loss': 0.4752, 'grad_norm': 17.37781524658203, 'learning_rate': 1.450406283979521e-07, 'epoch': 0.21263900189856252, 'num_input_tokens_seen': 6677331968, 'completed': '86.36% (3_184 / 3_687)', 'remaining time': '4:27:03', 'throughput': '8531.87', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 21:50:59,529 >> {'loss': 0.2446, 'grad_norm': 11.948087692260742, 'learning_rate': 1.4486475245170977e-07, 'epoch': 0.21291022511526986, 'num_input_tokens_seen': 6679429120, 'completed': '86.38% (3_185 / 3_687)', 'remaining time': '4:26:32', 'throughput': '8175.14', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 21:51:31,145 >> {'loss': 0.3559, 'grad_norm': 17.137893676757812, 'learning_rate': 1.446892025428981e-07, 'epoch': 0.21318144833197722, 'num_input_tokens_seen': 6681526272, 'completed': '86.41% (3_186 / 3_687)', 'remaining time': '4:26:00', 'throughput': '8291.53', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 21:52:01,342 >> {'loss': 0.1706, 'grad_norm': 8.884380340576172, 'learning_rate': 1.4451397881279298e-07, 'epoch': 0.21345267154868458, 'num_input_tokens_seen': 6683623424, 'completed': '86.44% (3_187 / 3_687)', 'remaining time': '4:25:27', 'throughput': '8680.97', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 21:52:31,091 >> {'loss': 0.3805, 'grad_norm': 16.802011489868164, 'learning_rate': 1.4433908140240792e-07, 'epoch': 0.21372389476539191, 'num_input_tokens_seen': 6685720576, 'completed': '86.47% (3_188 / 3_687)', 'remaining time': '4:24:54', 'throughput': '8811.96', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 21:53:02,121 >> {'loss': 0.3829, 'grad_norm': 20.294599533081055, 'learning_rate': 1.4416451045249354e-07, 'epoch': 0.21399511798209928, 'num_input_tokens_seen': 6687817728, 'completed': '86.49% (3_189 / 3_687)', 'remaining time': '4:24:21', 'throughput': '8447.97', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 21:53:34,203 >> {'loss': 0.3615, 'grad_norm': 16.205463409423828, 'learning_rate': 1.43990266103538e-07, 'epoch': 0.2142663411988066, 'num_input_tokens_seen': 6689914880, 'completed': '86.52% (3_190 / 3_687)', 'remaining time': '4:23:50', 'throughput': '8171.23', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 21:54:04,489 >> {'loss': 0.3264, 'grad_norm': 17.31773567199707, 'learning_rate': 1.4381634849576644e-07, 'epoch': 0.21453756441551397, 'num_input_tokens_seen': 6692012032, 'completed': '86.55% (3_191 / 3_687)', 'remaining time': '4:23:17', 'throughput': '8655.45', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 21:54:35,020 >> {'loss': 0.3105, 'grad_norm': 13.5274658203125, 'learning_rate': 1.4364275776914112e-07, 'epoch': 0.21480878763222133, 'num_input_tokens_seen': 6694109184, 'completed': '86.57% (3_192 / 3_687)', 'remaining time': '4:22:44', 'throughput': '8586.17', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 21:55:07,619 >> {'loss': 0.3113, 'grad_norm': 12.372538566589355, 'learning_rate': 1.4346949406336136e-07, 'epoch': 0.21508001084892867, 'num_input_tokens_seen': 6696206336, 'completed': '86.60% (3_193 / 3_687)', 'remaining time': '4:22:13', 'throughput': '8041.62', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 21:55:37,670 >> {'loss': 0.2954, 'grad_norm': 12.951509475708008, 'learning_rate': 1.4329655751786318e-07, 'epoch': 0.21535123406563603, 'num_input_tokens_seen': 6698303488, 'completed': '86.63% (3_194 / 3_687)', 'remaining time': '4:21:40', 'throughput': '8723.08', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 21:56:08,015 >> {'loss': 0.4798, 'grad_norm': 16.507877349853516, 'learning_rate': 1.4312394827181925e-07, 'epoch': 0.21562245728234336, 'num_input_tokens_seen': 6700400640, 'completed': '86.66% (3_195 / 3_687)', 'remaining time': '4:21:07', 'throughput': '8639.03', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 21:56:41,187 >> {'loss': 0.3777, 'grad_norm': 15.880234718322754, 'learning_rate': 1.4295166646413898e-07, 'epoch': 0.21589368049905072, 'num_input_tokens_seen': 6702497792, 'completed': '86.68% (3_196 / 3_687)', 'remaining time': '4:20:36', 'throughput': '7902.60', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 21:57:11,356 >> {'loss': 0.3242, 'grad_norm': 15.432538986206055, 'learning_rate': 1.4277971223346825e-07, 'epoch': 0.21616490371575806, 'num_input_tokens_seen': 6704594944, 'completed': '86.71% (3_197 / 3_687)', 'remaining time': '4:20:03', 'throughput': '8689.15', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 21:57:43,327 >> {'loss': 0.4032, 'grad_norm': 18.914257049560547, 'learning_rate': 1.4260808571818917e-07, 'epoch': 0.21643612693246542, 'num_input_tokens_seen': 6706692096, 'completed': '86.74% (3_198 / 3_687)', 'remaining time': '4:19:31', 'throughput': '8199.23', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 21:58:15,547 >> {'loss': 0.3723, 'grad_norm': 17.466224670410156, 'learning_rate': 1.4243678705642027e-07, 'epoch': 0.21670735014917278, 'num_input_tokens_seen': 6708789248, 'completed': '86.76% (3_199 / 3_687)', 'remaining time': '4:19:00', 'throughput': '8136.13', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 21:58:42,988 >> {'loss': 0.4615, 'grad_norm': 19.855295181274414, 'learning_rate': 1.4226581638601635e-07, 'epoch': 0.2169785733658801, 'num_input_tokens_seen': 6710886400, 'completed': '86.79% (3_200 / 3_687)', 'remaining time': '4:18:25', 'throughput': '9553.02', 'gpu_mem_free': '5581MB'} +/scratch3/workspace/ctpham_umass_edu-ft/envs/prolong-final/lib/python3.10/site-packages/torch/distributed/fsdp/fully_sharded_data_parallel.py:689: FutureWarning: FSDP.state_dict_type() and FSDP.set_state_dict_type() are being deprecated. Please use APIs, get_state_dict() and set_state_dict(), which can support different parallelisms, FSDP1, FSDP2, DDP. API doc: https://pytorch.org/docs/stable/distributed.checkpoint.html#torch.distributed.checkpoint.state_dict.get_state_dict .Tutorial: https://pytorch.org/tutorials/recipes/distributed_checkpoint_recipe.html . + warnings.warn( +[INFO|trainer.py:3503] 2024-12-26 21:59:06,009 >> Saving model checkpoint to /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/checkpoint-3200 +[INFO|configuration_utils.py:472] 2024-12-26 21:59:06,012 >> Configuration saved in /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/checkpoint-3200/config.json +[INFO|configuration_utils.py:807] 2024-12-26 21:59:06,014 >> Configuration saved in /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/checkpoint-3200/generation_config.json +[INFO|modeling_utils.py:2807] 2024-12-26 22:00:00,600 >> The model is bigger than the maximum size per checkpoint (5GB) and is going to be split in 7 checkpoint shards. You can find where each parameters has been saved in the index located at /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/checkpoint-3200/model.safetensors.index.json. +[INFO|tokenization_utils_base.py:2684] 2024-12-26 22:00:00,604 >> tokenizer config file saved in /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/checkpoint-3200/tokenizer_config.json +[INFO|tokenization_utils_base.py:2693] 2024-12-26 22:00:00,605 >> Special tokens file saved in /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/checkpoint-3200/special_tokens_map.json +/scratch3/workspace/ctpham_umass_edu-ft/envs/prolong-final/lib/python3.10/site-packages/torch/distributed/fsdp/fully_sharded_data_parallel.py:689: FutureWarning: FSDP.state_dict_type() and FSDP.set_state_dict_type() are being deprecated. Please use APIs, get_state_dict() and set_state_dict(), which can support different parallelisms, FSDP1, FSDP2, DDP. API doc: https://pytorch.org/docs/stable/distributed.checkpoint.html#torch.distributed.checkpoint.state_dict.get_state_dict .Tutorial: https://pytorch.org/tutorials/recipes/distributed_checkpoint_recipe.html . + warnings.warn( +[WARNING|trainer.py:868] 2024-12-26 22:03:38,362 >> Save streaming dataset state: {'epoch': 0, 'sample_in_epoch': 6400, 'num_canonical_nodes': 1, 'shuffle_seed': 42, 'initial_physical_nodes': 1} +/scratch3/workspace/ctpham_umass_edu-ft/envs/prolong-final/lib/python3.10/site-packages/torch/utils/checkpoint.py:1399: FutureWarning: `torch.cpu.amp.autocast(args...)` is deprecated. Please use `torch.amp.autocast('cpu', args...)` instead. + with device_autocast_ctx, torch.cpu.amp.autocast(**cpu_autocast_kwargs), recompute_context: # type: ignore[attr-defined] +[INFO|trainer.py:175] 2024-12-26 22:04:11,264 >> {'loss': 0.5931, 'grad_norm': 20.14664077758789, 'learning_rate': 1.4209517384456786e-07, 'epoch': 0.21724979658258747, 'num_input_tokens_seen': 6712983552, 'completed': '86.82% (3_201 / 3_687)', 'remaining time': '4:20:53', 'throughput': '798.55', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 22:04:43,013 >> {'loss': 0.3161, 'grad_norm': 11.748592376708984, 'learning_rate': 1.4192485956940153e-07, 'epoch': 0.2175210197992948, 'num_input_tokens_seen': 6715080704, 'completed': '86.85% (3_202 / 3_687)', 'remaining time': '4:20:21', 'throughput': '8256.72', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 22:05:15,795 >> {'loss': 0.2212, 'grad_norm': 10.327018737792969, 'learning_rate': 1.4175487369757973e-07, 'epoch': 0.21779224301600217, 'num_input_tokens_seen': 6717177856, 'completed': '86.87% (3_203 / 3_687)', 'remaining time': '4:19:49', 'throughput': '7996.70', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 22:05:45,393 >> {'loss': 0.5347, 'grad_norm': 18.968761444091797, 'learning_rate': 1.4158521636590079e-07, 'epoch': 0.21806346623270953, 'num_input_tokens_seen': 6719275008, 'completed': '86.90% (3_204 / 3_687)', 'remaining time': '4:19:15', 'throughput': '8856.67', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 22:06:13,516 >> {'loss': 0.6773, 'grad_norm': 20.50962257385254, 'learning_rate': 1.4141588771089838e-07, 'epoch': 0.21833468944941686, 'num_input_tokens_seen': 6721372160, 'completed': '86.93% (3_205 / 3_687)', 'remaining time': '4:18:40', 'throughput': '9321.50', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 22:06:44,712 >> {'loss': 0.2704, 'grad_norm': 11.771401405334473, 'learning_rate': 1.412468878688418e-07, 'epoch': 0.21860591266612422, 'num_input_tokens_seen': 6723469312, 'completed': '86.95% (3_206 / 3_687)', 'remaining time': '4:18:08', 'throughput': '8402.90', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 22:07:14,889 >> {'loss': 0.2279, 'grad_norm': 11.137237548828125, 'learning_rate': 1.410782169757356e-07, 'epoch': 0.21887713588283156, 'num_input_tokens_seen': 6725566464, 'completed': '86.98% (3_207 / 3_687)', 'remaining time': '4:17:34', 'throughput': '8687.04', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 22:07:45,137 >> {'loss': 0.3856, 'grad_norm': 14.032715797424316, 'learning_rate': 1.4090987516731977e-07, 'epoch': 0.21914835909953892, 'num_input_tokens_seen': 6727663616, 'completed': '87.01% (3_208 / 3_687)', 'remaining time': '4:17:01', 'throughput': '8666.32', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 22:08:18,116 >> {'loss': 0.341, 'grad_norm': 13.834157943725586, 'learning_rate': 1.4074186257906934e-07, 'epoch': 0.21941958231624628, 'num_input_tokens_seen': 6729760768, 'completed': '87.04% (3_209 / 3_687)', 'remaining time': '4:16:29', 'throughput': '7948.84', 'gpu_mem_free': '5581MB'}