diff --git "a/wandb/run-20241225_152744-sac86tlv/files/output.log" "b/wandb/run-20241225_152744-sac86tlv/files/output.log" new file mode 100644--- /dev/null +++ "b/wandb/run-20241225_152744-sac86tlv/files/output.log" @@ -0,0 +1,1854 @@ +/scratch3/workspace/ctpham_umass_edu-ft/envs/prolong-final/lib/python3.10/site-packages/transformers/trainer.py:2833: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. + checkpoint_rng_state = torch.load(rng_file) +/scratch3/workspace/ctpham_umass_edu-ft/envs/prolong-final/lib/python3.10/site-packages/torch/utils/checkpoint.py:1399: FutureWarning: `torch.cpu.amp.autocast(args...)` is deprecated. Please use `torch.amp.autocast('cpu', args...)` instead. + with device_autocast_ctx, torch.cpu.amp.autocast(**cpu_autocast_kwargs), recompute_context: # type: ignore[attr-defined] +[INFO|trainer.py:175] 2024-12-25 15:28:35,986 >> {'loss': 0.3659, 'grad_norm': 14.524723052978516, 'learning_rate': 9.33022124140483e-07, 'epoch': 0.00027122321670735016, 'num_input_tokens_seen': 1679818752, 'completed': '21.72% (801 / 3_687)', 'remaining time': '34:45:09', 'throughput': '3023.54', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 15:29:06,898 >> {'loss': 0.3889, 'grad_norm': 12.789175987243652, 'learning_rate': 9.32810071644894e-07, 'epoch': 0.0005424464334147003, 'num_input_tokens_seen': 1681915904, 'completed': '21.75% (802 / 3_687)', 'remaining time': '29:45:24', 'throughput': '8480.25', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 15:29:37,539 >> {'loss': 0.3584, 'grad_norm': 15.237256050109863, 'learning_rate': 9.325977110783263e-07, 'epoch': 0.0008136696501220504, 'num_input_tokens_seen': 1684013056, 'completed': '21.78% (803 / 3_687)', 'remaining time': '28:00:47', 'throughput': '8555.47', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 15:30:04,732 >> {'loss': 0.5716, 'grad_norm': 17.71392250061035, 'learning_rate': 9.323850426116797e-07, 'epoch': 0.0010848928668294006, 'num_input_tokens_seen': 1686110208, 'completed': '21.81% (804 / 3_687)', 'remaining time': '26:26:48', 'throughput': '9640.04', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 15:30:36,798 >> {'loss': 0.5411, 'grad_norm': 20.835195541381836, 'learning_rate': 9.321720664161017e-07, 'epoch': 0.0013561160835367507, 'num_input_tokens_seen': 1688207360, 'completed': '21.83% (805 / 3_687)', 'remaining time': '26:17:02', 'throughput': '8175.29', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 15:31:05,593 >> {'loss': 0.5051, 'grad_norm': 20.98845100402832, 'learning_rate': 9.319587826629872e-07, 'epoch': 0.0016273393002441008, 'num_input_tokens_seen': 1690304512, 'completed': '21.86% (806 / 3_687)', 'remaining time': '25:44:11', 'throughput': '9103.82', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 15:31:37,803 >> {'loss': 0.6283, 'grad_norm': 17.86595916748047, 'learning_rate': 9.317451915239792e-07, 'epoch': 0.001898562516951451, 'num_input_tokens_seen': 1692401664, 'completed': '21.89% (807 / 3_687)', 'remaining time': '25:44:00', 'throughput': '8138.46', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 15:32:10,895 >> {'loss': 0.5475, 'grad_norm': 19.295974731445312, 'learning_rate': 9.315312931709674e-07, 'epoch': 0.0021697857336588013, 'num_input_tokens_seen': 1694498816, 'completed': '21.91% (808 / 3_687)', 'remaining time': '25:49:00', 'throughput': '7921.76', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 15:32:45,862 >> {'loss': 0.6961, 'grad_norm': 20.173925399780273, 'learning_rate': 9.313170877760892e-07, 'epoch': 0.0024410089503661514, 'num_input_tokens_seen': 1696595968, 'completed': '21.94% (809 / 3_687)', 'remaining time': '26:02:47', 'throughput': '7496.72', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 15:33:16,398 >> {'loss': 0.3755, 'grad_norm': 13.733884811401367, 'learning_rate': 9.311025755117291e-07, 'epoch': 0.0027122321670735015, 'num_input_tokens_seen': 1698693120, 'completed': '21.97% (810 / 3_687)', 'remaining time': '25:52:26', 'throughput': '8584.93', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 15:33:47,984 >> {'loss': 0.6359, 'grad_norm': 19.169233322143555, 'learning_rate': 9.308877565505181e-07, 'epoch': 0.0029834553837808516, 'num_input_tokens_seen': 1700790272, 'completed': '22.00% (811 / 3_687)', 'remaining time': '25:48:27', 'throughput': '8299.33', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 15:34:19,597 >> {'loss': 0.3631, 'grad_norm': 16.250288009643555, 'learning_rate': 9.306726310653346e-07, 'epoch': 0.0032546786004882017, 'num_input_tokens_seen': 1702887424, 'completed': '22.02% (812 / 3_687)', 'remaining time': '25:45:09', 'throughput': '8292.22', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 15:34:50,554 >> {'loss': 0.672, 'grad_norm': 34.65564727783203, 'learning_rate': 9.304571992293032e-07, 'epoch': 0.003525901817195552, 'num_input_tokens_seen': 1704984576, 'completed': '22.05% (813 / 3_687)', 'remaining time': '25:39:52', 'throughput': '8468.13', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 15:35:21,194 >> {'loss': 0.9443, 'grad_norm': 30.45555877685547, 'learning_rate': 9.302414612157954e-07, 'epoch': 0.003797125033902902, 'num_input_tokens_seen': 1707081728, 'completed': '22.08% (814 / 3_687)', 'remaining time': '25:34:10', 'throughput': '8555.66', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 15:35:50,992 >> {'loss': 0.5574, 'grad_norm': 19.15201759338379, 'learning_rate': 9.300254171984289e-07, 'epoch': 0.0040683482506102524, 'num_input_tokens_seen': 1709178880, 'completed': '22.10% (815 / 3_687)', 'remaining time': '25:26:29', 'throughput': '8797.37', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 15:36:24,017 >> {'loss': 0.724, 'grad_norm': 29.86651039123535, 'learning_rate': 9.298090673510677e-07, 'epoch': 0.0043395714673176026, 'num_input_tokens_seen': 1711276032, 'completed': '22.13% (816 / 3_687)', 'remaining time': '25:29:20', 'throughput': '7937.74', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 15:36:56,450 >> {'loss': 0.6076, 'grad_norm': 18.136457443237305, 'learning_rate': 9.295924118478218e-07, 'epoch': 0.004610794684024953, 'num_input_tokens_seen': 1713373184, 'completed': '22.16% (817 / 3_687)', 'remaining time': '25:30:08', 'throughput': '8082.53', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 15:37:23,348 >> {'loss': 0.7823, 'grad_norm': 19.57548713684082, 'learning_rate': 9.293754508630473e-07, 'epoch': 0.004882017900732303, 'num_input_tokens_seen': 1715470336, 'completed': '22.19% (818 / 3_687)', 'remaining time': '25:16:05', 'throughput': '9745.83', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 15:37:52,391 >> {'loss': 0.4778, 'grad_norm': 14.906988143920898, 'learning_rate': 9.291581845713466e-07, 'epoch': 0.005153241117439653, 'num_input_tokens_seen': 1717567488, 'completed': '22.21% (819 / 3_687)', 'remaining time': '25:08:51', 'throughput': '9026.07', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 15:38:24,390 >> {'loss': 0.3655, 'grad_norm': 14.388298034667969, 'learning_rate': 9.289406131475665e-07, 'epoch': 0.005424464334147003, 'num_input_tokens_seen': 1719664640, 'completed': '22.24% (820 / 3_687)', 'remaining time': '25:09:21', 'throughput': '8192.32', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 15:38:53,145 >> {'loss': 0.6397, 'grad_norm': 19.326364517211914, 'learning_rate': 9.287227367668012e-07, 'epoch': 0.005695687550854353, 'num_input_tokens_seen': 1721761792, 'completed': '22.27% (821 / 3_687)', 'remaining time': '25:02:23', 'throughput': '9116.55', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 15:39:26,625 >> {'loss': 0.3671, 'grad_norm': 12.356776237487793, 'learning_rate': 9.285045556043885e-07, 'epoch': 0.005966910767561703, 'num_input_tokens_seen': 1723858944, 'completed': '22.29% (822 / 3_687)', 'remaining time': '25:06:16', 'throughput': '7829.66', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 15:39:53,398 >> {'loss': 0.763, 'grad_norm': 18.93061065673828, 'learning_rate': 9.282860698359128e-07, 'epoch': 0.006238133984269053, 'num_input_tokens_seen': 1725956096, 'completed': '22.32% (823 / 3_687)', 'remaining time': '24:55:50', 'throughput': '9791.56', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 15:40:20,749 >> {'loss': 0.9767, 'grad_norm': 23.214996337890625, 'learning_rate': 9.280672796372029e-07, 'epoch': 0.006509357200976403, 'num_input_tokens_seen': 1728053248, 'completed': '22.35% (824 / 3_687)', 'remaining time': '24:47:23', 'throughput': '9584.33', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 15:40:49,260 >> {'loss': 0.6971, 'grad_norm': 21.05919075012207, 'learning_rate': 9.278481851843327e-07, 'epoch': 0.0067805804176837535, 'num_input_tokens_seen': 1730150400, 'completed': '22.38% (825 / 3_687)', 'remaining time': '24:41:48', 'throughput': '9194.39', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 15:41:20,565 >> {'loss': 0.6836, 'grad_norm': 20.329505920410156, 'learning_rate': 9.276287866536215e-07, 'epoch': 0.007051803634391104, 'num_input_tokens_seen': 1732247552, 'completed': '22.40% (826 / 3_687)', 'remaining time': '24:41:43', 'throughput': '8374.10', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 15:41:50,661 >> {'loss': 0.3354, 'grad_norm': 13.536745071411133, 'learning_rate': 9.274090842216326e-07, 'epoch': 0.007323026851098454, 'num_input_tokens_seen': 1734344704, 'completed': '22.43% (827 / 3_687)', 'remaining time': '24:39:28', 'throughput': '8710.18', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 15:42:22,992 >> {'loss': 0.648, 'grad_norm': 19.788000106811523, 'learning_rate': 9.271890780651741e-07, 'epoch': 0.007594250067805804, 'num_input_tokens_seen': 1736441856, 'completed': '22.46% (828 / 3_687)', 'remaining time': '24:41:09', 'throughput': '8108.01', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 15:42:54,138 >> {'loss': 0.2731, 'grad_norm': 11.306535720825195, 'learning_rate': 9.269687683612987e-07, 'epoch': 0.007865473284513154, 'num_input_tokens_seen': 1738539008, 'completed': '22.48% (829 / 3_687)', 'remaining time': '24:40:44', 'throughput': '8416.76', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 15:43:23,485 >> {'loss': 0.4084, 'grad_norm': 20.76605796813965, 'learning_rate': 9.267481552873033e-07, 'epoch': 0.008136696501220505, 'num_input_tokens_seen': 1740636160, 'completed': '22.51% (830 / 3_687)', 'remaining time': '24:37:27', 'throughput': '8932.63', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 15:43:53,981 >> {'loss': 0.4862, 'grad_norm': 17.467897415161133, 'learning_rate': 9.265272390207289e-07, 'epoch': 0.008407919717927854, 'num_input_tokens_seen': 1742733312, 'completed': '22.54% (831 / 3_687)', 'remaining time': '24:36:07', 'throughput': '8596.04', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 15:44:22,284 >> {'loss': 0.5836, 'grad_norm': 21.455224990844727, 'learning_rate': 9.263060197393603e-07, 'epoch': 0.008679142934635205, 'num_input_tokens_seen': 1744830464, 'completed': '22.57% (832 / 3_687)', 'remaining time': '24:31:35', 'throughput': '9262.05', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 15:44:49,520 >> {'loss': 0.9138, 'grad_norm': 22.814332962036133, 'learning_rate': 9.260844976212268e-07, 'epoch': 0.008950366151342554, 'num_input_tokens_seen': 1746927616, 'completed': '22.59% (833 / 3_687)', 'remaining time': '24:25:45', 'throughput': '9624.88', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 15:45:22,809 >> {'loss': 0.3955, 'grad_norm': 13.567204475402832, 'learning_rate': 9.258626728446004e-07, 'epoch': 0.009221589368049905, 'num_input_tokens_seen': 1749024768, 'completed': '22.62% (834 / 3_687)', 'remaining time': '24:28:41', 'throughput': '7874.65', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 15:45:50,201 >> {'loss': 0.7177, 'grad_norm': 19.025236129760742, 'learning_rate': 9.256405455879977e-07, 'epoch': 0.009492812584757255, 'num_input_tokens_seen': 1751121920, 'completed': '22.65% (835 / 3_687)', 'remaining time': '24:23:26', 'throughput': '9570.26', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 15:46:19,510 >> {'loss': 0.6642, 'grad_norm': 22.544050216674805, 'learning_rate': 9.25418116030178e-07, 'epoch': 0.009764035801464606, 'num_input_tokens_seen': 1753219072, 'completed': '22.67% (836 / 3_687)', 'remaining time': '24:20:58', 'throughput': '8944.18', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 15:46:50,922 >> {'loss': 0.3442, 'grad_norm': 13.098361015319824, 'learning_rate': 9.251953843501443e-07, 'epoch': 0.010035259018171955, 'num_input_tokens_seen': 1755316224, 'completed': '22.70% (837 / 3_687)', 'remaining time': '24:21:18', 'throughput': '8345.31', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 15:47:21,394 >> {'loss': 0.5423, 'grad_norm': 20.6568660736084, 'learning_rate': 9.249723507271425e-07, 'epoch': 0.010306482234879306, 'num_input_tokens_seen': 1757413376, 'completed': '22.73% (838 / 3_687)', 'remaining time': '24:20:26', 'throughput': '8602.70', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 15:47:51,979 >> {'loss': 0.5084, 'grad_norm': 17.13654899597168, 'learning_rate': 9.247490153406617e-07, 'epoch': 0.010577705451586655, 'num_input_tokens_seen': 1759510528, 'completed': '22.76% (839 / 3_687)', 'remaining time': '24:19:42', 'throughput': '8570.94', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 15:48:23,595 >> {'loss': 0.3407, 'grad_norm': 16.935338973999023, 'learning_rate': 9.245253783704334e-07, 'epoch': 0.010848928668294006, 'num_input_tokens_seen': 1761607680, 'completed': '22.78% (840 / 3_687)', 'remaining time': '24:20:13', 'throughput': '8291.62', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 15:48:53,372 >> {'loss': 0.3577, 'grad_norm': 12.196868896484375, 'learning_rate': 9.243014399964324e-07, 'epoch': 0.011120151885001357, 'num_input_tokens_seen': 1763704832, 'completed': '22.81% (841 / 3_687)', 'remaining time': '24:18:33', 'throughput': '8803.49', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 15:49:24,349 >> {'loss': 0.3152, 'grad_norm': 10.492193222045898, 'learning_rate': 9.240772003988758e-07, 'epoch': 0.011391375101708706, 'num_input_tokens_seen': 1765801984, 'completed': '22.84% (842 / 3_687)', 'remaining time': '24:18:18', 'throughput': '8462.62', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 15:49:54,691 >> {'loss': 0.4958, 'grad_norm': 17.036666870117188, 'learning_rate': 9.238526597582229e-07, 'epoch': 0.011662598318416057, 'num_input_tokens_seen': 1767899136, 'completed': '22.86% (843 / 3_687)', 'remaining time': '24:17:20', 'throughput': '8639.70', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 15:50:23,631 >> {'loss': 0.6916, 'grad_norm': 26.499019622802734, 'learning_rate': 9.236278182551758e-07, 'epoch': 0.011933821535123406, 'num_input_tokens_seen': 1769996288, 'completed': '22.89% (844 / 3_687)', 'remaining time': '24:14:52', 'throughput': '9058.05', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 15:50:51,039 >> {'loss': 0.5991, 'grad_norm': 18.293750762939453, 'learning_rate': 9.23402676070678e-07, 'epoch': 0.012205044751830757, 'num_input_tokens_seen': 1772093440, 'completed': '22.92% (845 / 3_687)', 'remaining time': '24:10:53', 'throughput': '9564.52', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 15:51:22,677 >> {'loss': 0.6295, 'grad_norm': 19.971363067626953, 'learning_rate': 9.231772333859154e-07, 'epoch': 0.012476267968538107, 'num_input_tokens_seen': 1774190592, 'completed': '22.95% (846 / 3_687)', 'remaining time': '24:11:25', 'throughput': '8285.82', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 15:51:52,254 >> {'loss': 0.3458, 'grad_norm': 15.316537857055664, 'learning_rate': 9.22951490382316e-07, 'epoch': 0.012747491185245458, 'num_input_tokens_seen': 1776287744, 'completed': '22.97% (847 / 3_687)', 'remaining time': '24:09:49', 'throughput': '8863.05', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 15:52:23,039 >> {'loss': 0.3044, 'grad_norm': 13.23933219909668, 'learning_rate': 9.22725447241549e-07, 'epoch': 0.013018714401952807, 'num_input_tokens_seen': 1778384896, 'completed': '23.00% (848 / 3_687)', 'remaining time': '24:09:28', 'throughput': '8515.28', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 15:52:56,342 >> {'loss': 0.4336, 'grad_norm': 17.797571182250977, 'learning_rate': 9.224991041455252e-07, 'epoch': 0.013289937618660158, 'num_input_tokens_seen': 1780482048, 'completed': '23.03% (849 / 3_687)', 'remaining time': '24:11:32', 'throughput': '7871.39', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 15:53:26,337 >> {'loss': 0.6113, 'grad_norm': 17.58323860168457, 'learning_rate': 9.222724612763971e-07, 'epoch': 0.013561160835367507, 'num_input_tokens_seen': 1782579200, 'completed': '23.05% (850 / 3_687)', 'remaining time': '24:10:22', 'throughput': '8739.74', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 15:53:54,885 >> {'loss': 0.6039, 'grad_norm': 17.832801818847656, 'learning_rate': 9.220455188165582e-07, 'epoch': 0.013832384052074858, 'num_input_tokens_seen': 1784676352, 'completed': '23.08% (851 / 3_687)', 'remaining time': '24:07:53', 'throughput': '9182.49', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 15:54:24,703 >> {'loss': 0.7489, 'grad_norm': 23.844926834106445, 'learning_rate': 9.218182769486433e-07, 'epoch': 0.014103607268782207, 'num_input_tokens_seen': 1786773504, 'completed': '23.11% (852 / 3_687)', 'remaining time': '24:06:38', 'throughput': '8791.40', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 15:54:52,313 >> {'loss': 0.615, 'grad_norm': 16.524341583251953, 'learning_rate': 9.215907358555276e-07, 'epoch': 0.014374830485489558, 'num_input_tokens_seen': 1788870656, 'completed': '23.14% (853 / 3_687)', 'remaining time': '24:03:26', 'throughput': '9494.59', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 15:55:24,214 >> {'loss': 0.867, 'grad_norm': 24.72736358642578, 'learning_rate': 9.213628957203277e-07, 'epoch': 0.014646053702196907, 'num_input_tokens_seen': 1790967808, 'completed': '23.16% (854 / 3_687)', 'remaining time': '24:04:06', 'throughput': '8217.57', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 15:55:54,594 >> {'loss': 0.8105, 'grad_norm': 25.9926700592041, 'learning_rate': 9.21134756726401e-07, 'epoch': 0.014917276918904258, 'num_input_tokens_seen': 1793064960, 'completed': '23.19% (855 / 3_687)', 'remaining time': '24:03:25', 'throughput': '8628.67', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 15:56:25,075 >> {'loss': 0.5625, 'grad_norm': 20.51833724975586, 'learning_rate': 9.209063190573445e-07, 'epoch': 0.015188500135611608, 'num_input_tokens_seen': 1795162112, 'completed': '23.22% (856 / 3_687)', 'remaining time': '24:02:49', 'throughput': '8600.28', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 15:56:57,529 >> {'loss': 0.4131, 'grad_norm': 15.484442710876465, 'learning_rate': 9.206775828969967e-07, 'epoch': 0.015459723352318959, 'num_input_tokens_seen': 1797259264, 'completed': '23.24% (857 / 3_687)', 'remaining time': '24:03:52', 'throughput': '8077.55', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 15:57:29,277 >> {'loss': 0.3329, 'grad_norm': 12.139299392700195, 'learning_rate': 9.204485484294355e-07, 'epoch': 0.015730946569026308, 'num_input_tokens_seen': 1799356416, 'completed': '23.27% (858 / 3_687)', 'remaining time': '24:04:17', 'throughput': '8256.88', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 15:57:59,433 >> {'loss': 0.5659, 'grad_norm': 15.683771133422852, 'learning_rate': 9.202192158389791e-07, 'epoch': 0.01600216978573366, 'num_input_tokens_seen': 1801453568, 'completed': '23.30% (859 / 3_687)', 'remaining time': '24:03:23', 'throughput': '8693.03', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 15:58:30,295 >> {'loss': 0.6709, 'grad_norm': 21.66155242919922, 'learning_rate': 9.199895853101856e-07, 'epoch': 0.01627339300244101, 'num_input_tokens_seen': 1803550720, 'completed': '23.33% (860 / 3_687)', 'remaining time': '24:03:04', 'throughput': '8493.97', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 15:58:58,447 >> {'loss': 0.7169, 'grad_norm': 19.560569763183594, 'learning_rate': 9.197596570278529e-07, 'epoch': 0.01654461621914836, 'num_input_tokens_seen': 1805647872, 'completed': '23.35% (861 / 3_687)', 'remaining time': '24:00:39', 'throughput': '9311.76', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 15:59:28,555 >> {'loss': 0.3676, 'grad_norm': 14.26892375946045, 'learning_rate': 9.19529431177019e-07, 'epoch': 0.016815839435855708, 'num_input_tokens_seen': 1807745024, 'completed': '23.38% (862 / 3_687)', 'remaining time': '23:59:46', 'throughput': '8706.71', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 15:59:57,537 >> {'loss': 0.1885, 'grad_norm': 9.662439346313477, 'learning_rate': 9.192989079429603e-07, 'epoch': 0.01708706265256306, 'num_input_tokens_seen': 1809842176, 'completed': '23.41% (863 / 3_687)', 'remaining time': '23:58:04', 'throughput': '9045.20', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 16:00:28,114 >> {'loss': 0.4003, 'grad_norm': 16.50687026977539, 'learning_rate': 9.190680875111934e-07, 'epoch': 0.01735828586927041, 'num_input_tokens_seen': 1811939328, 'completed': '23.43% (864 / 3_687)', 'remaining time': '23:57:34', 'throughput': '8573.20', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 16:01:01,622 >> {'loss': 0.4239, 'grad_norm': 16.06983757019043, 'learning_rate': 9.188369700674735e-07, 'epoch': 0.01762950908597776, 'num_input_tokens_seen': 1814036480, 'completed': '23.46% (865 / 3_687)', 'remaining time': '23:59:12', 'throughput': '7823.39', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 16:01:32,933 >> {'loss': 0.5002, 'grad_norm': 17.57181739807129, 'learning_rate': 9.186055557977957e-07, 'epoch': 0.01790073230268511, 'num_input_tokens_seen': 1816133632, 'completed': '23.49% (866 / 3_687)', 'remaining time': '23:59:12', 'throughput': '8372.29', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 16:02:01,295 >> {'loss': 0.6752, 'grad_norm': 22.786888122558594, 'learning_rate': 9.18373844888393e-07, 'epoch': 0.01817195551939246, 'num_input_tokens_seen': 1818230784, 'completed': '23.52% (867 / 3_687)', 'remaining time': '23:57:07', 'throughput': '9242.83', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 16:02:32,492 >> {'loss': 0.477, 'grad_norm': 15.312600135803223, 'learning_rate': 9.181418375257374e-07, 'epoch': 0.01844317873609981, 'num_input_tokens_seen': 1820327936, 'completed': '23.54% (868 / 3_687)', 'remaining time': '23:57:02', 'throughput': '8402.63', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 16:03:00,117 >> {'loss': 0.6745, 'grad_norm': 21.413515090942383, 'learning_rate': 9.179095338965401e-07, 'epoch': 0.01871440195280716, 'num_input_tokens_seen': 1822425088, 'completed': '23.57% (869 / 3_687)', 'remaining time': '23:54:30', 'throughput': '9489.46', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 16:03:31,859 >> {'loss': 0.4547, 'grad_norm': 14.375008583068848, 'learning_rate': 9.176769341877497e-07, 'epoch': 0.01898562516951451, 'num_input_tokens_seen': 1824522240, 'completed': '23.60% (870 / 3_687)', 'remaining time': '23:54:48', 'throughput': '8258.69', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 16:04:00,965 >> {'loss': 0.5549, 'grad_norm': 18.940818786621094, 'learning_rate': 9.17444038586554e-07, 'epoch': 0.01925684838622186, 'num_input_tokens_seen': 1826619392, 'completed': '23.62% (871 / 3_687)', 'remaining time': '23:53:20', 'throughput': '9006.42', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 16:04:31,371 >> {'loss': 0.6916, 'grad_norm': 22.97831916809082, 'learning_rate': 9.172108472803782e-07, 'epoch': 0.01952807160292921, 'num_input_tokens_seen': 1828716544, 'completed': '23.65% (872 / 3_687)', 'remaining time': '23:52:44', 'throughput': '8621.39', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 16:05:03,161 >> {'loss': 0.464, 'grad_norm': 33.489131927490234, 'learning_rate': 9.16977360456886e-07, 'epoch': 0.019799294819636562, 'num_input_tokens_seen': 1830813696, 'completed': '23.68% (873 / 3_687)', 'remaining time': '23:53:02', 'throughput': '8246.14', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 16:05:35,377 >> {'loss': 0.3587, 'grad_norm': 12.543787002563477, 'learning_rate': 9.167435783039786e-07, 'epoch': 0.02007051803634391, 'num_input_tokens_seen': 1832910848, 'completed': '23.70% (874 / 3_687)', 'remaining time': '23:53:34', 'throughput': '8137.22', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 16:06:04,922 >> {'loss': 0.5598, 'grad_norm': 16.099515914916992, 'learning_rate': 9.165095010097949e-07, 'epoch': 0.02034174125305126, 'num_input_tokens_seen': 1835008000, 'completed': '23.73% (875 / 3_687)', 'remaining time': '23:52:25', 'throughput': '8872.72', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 16:06:35,393 >> {'loss': 0.645, 'grad_norm': 18.598281860351562, 'learning_rate': 9.162751287627116e-07, 'epoch': 0.02061296446975861, 'num_input_tokens_seen': 1837105152, 'completed': '23.76% (876 / 3_687)', 'remaining time': '23:51:51', 'throughput': '8603.05', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 16:07:05,974 >> {'loss': 0.4911, 'grad_norm': 14.963810920715332, 'learning_rate': 9.160404617513424e-07, 'epoch': 0.020884187686465962, 'num_input_tokens_seen': 1839202304, 'completed': '23.79% (877 / 3_687)', 'remaining time': '23:51:21', 'throughput': '8572.13', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 16:07:34,179 >> {'loss': 0.8007, 'grad_norm': 21.489377975463867, 'learning_rate': 9.158055001645385e-07, 'epoch': 0.02115541090317331, 'num_input_tokens_seen': 1841299456, 'completed': '23.81% (878 / 3_687)', 'remaining time': '23:49:26', 'throughput': '9293.98', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 16:08:05,242 >> {'loss': 0.4235, 'grad_norm': 14.658339500427246, 'learning_rate': 9.155702441913881e-07, 'epoch': 0.02142663411988066, 'num_input_tokens_seen': 1843396608, 'completed': '23.84% (879 / 3_687)', 'remaining time': '23:49:14', 'throughput': '8439.10', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 16:08:35,579 >> {'loss': 0.5487, 'grad_norm': 20.184280395507812, 'learning_rate': 9.15334694021216e-07, 'epoch': 0.021697857336588012, 'num_input_tokens_seen': 1845493760, 'completed': '23.87% (880 / 3_687)', 'remaining time': '23:48:36', 'throughput': '8641.18', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 16:09:04,734 >> {'loss': 0.558, 'grad_norm': 17.544336318969727, 'learning_rate': 9.150988498435843e-07, 'epoch': 0.021969080553295363, 'num_input_tokens_seen': 1847590912, 'completed': '23.89% (881 / 3_687)', 'remaining time': '23:47:18', 'throughput': '8991.35', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 16:09:37,464 >> {'loss': 0.3181, 'grad_norm': 12.21959400177002, 'learning_rate': 9.148627118482912e-07, 'epoch': 0.022240303770002714, 'num_input_tokens_seen': 1849688064, 'completed': '23.92% (882 / 3_687)', 'remaining time': '23:48:03', 'throughput': '8009.24', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 16:10:05,401 >> {'loss': 0.7004, 'grad_norm': 17.65806007385254, 'learning_rate': 9.146262802253717e-07, 'epoch': 0.02251152698671006, 'num_input_tokens_seen': 1851785216, 'completed': '23.95% (883 / 3_687)', 'remaining time': '23:46:04', 'throughput': '9383.42', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 16:10:36,561 >> {'loss': 0.3288, 'grad_norm': 11.67097282409668, 'learning_rate': 9.14389555165097e-07, 'epoch': 0.022782750203417412, 'num_input_tokens_seen': 1853882368, 'completed': '23.98% (884 / 3_687)', 'remaining time': '23:45:55', 'throughput': '8412.94', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 16:11:08,831 >> {'loss': 0.4907, 'grad_norm': 17.335294723510742, 'learning_rate': 9.141525368579742e-07, 'epoch': 0.023053973420124763, 'num_input_tokens_seen': 1855979520, 'completed': '24.00% (885 / 3_687)', 'remaining time': '23:46:22', 'throughput': '8123.32', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 16:11:40,256 >> {'loss': 0.424, 'grad_norm': 21.265504837036133, 'learning_rate': 9.139152254947469e-07, 'epoch': 0.023325196636832114, 'num_input_tokens_seen': 1858076672, 'completed': '24.03% (886 / 3_687)', 'remaining time': '23:46:20', 'throughput': '8341.93', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 16:12:13,263 >> {'loss': 0.4297, 'grad_norm': 12.256847381591797, 'learning_rate': 9.136776212663942e-07, 'epoch': 0.023596419853539462, 'num_input_tokens_seen': 1860173824, 'completed': '24.06% (887 / 3_687)', 'remaining time': '23:47:09', 'throughput': '7942.07', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 16:12:43,718 >> {'loss': 0.6221, 'grad_norm': 20.520448684692383, 'learning_rate': 9.134397243641307e-07, 'epoch': 0.023867643070246813, 'num_input_tokens_seen': 1862270976, 'completed': '24.08% (888 / 3_687)', 'remaining time': '23:46:34', 'throughput': '8607.72', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 16:13:16,508 >> {'loss': 0.3873, 'grad_norm': 15.912782669067383, 'learning_rate': 9.132015349794069e-07, 'epoch': 0.024138866286954164, 'num_input_tokens_seen': 1864368128, 'completed': '24.11% (889 / 3_687)', 'remaining time': '23:47:13', 'throughput': '7994.47', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 16:13:48,305 >> {'loss': 0.2584, 'grad_norm': 12.093875885009766, 'learning_rate': 9.129630533039086e-07, 'epoch': 0.024410089503661515, 'num_input_tokens_seen': 1866465280, 'completed': '24.14% (890 / 3_687)', 'remaining time': '23:47:20', 'throughput': '8244.42', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 16:14:20,405 >> {'loss': 0.7337, 'grad_norm': 25.329559326171875, 'learning_rate': 9.127242795295569e-07, 'epoch': 0.024681312720368862, 'num_input_tokens_seen': 1868562432, 'completed': '24.17% (891 / 3_687)', 'remaining time': '23:47:34', 'throughput': '8166.42', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 16:14:49,995 >> {'loss': 0.5747, 'grad_norm': 23.77729034423828, 'learning_rate': 9.124852138485076e-07, 'epoch': 0.024952535937076213, 'num_input_tokens_seen': 1870659584, 'completed': '24.19% (892 / 3_687)', 'remaining time': '23:46:32', 'throughput': '8859.19', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 16:15:21,394 >> {'loss': 0.8033, 'grad_norm': 24.904544830322266, 'learning_rate': 9.12245856453152e-07, 'epoch': 0.025223759153783564, 'num_input_tokens_seen': 1872756736, 'completed': '24.22% (893 / 3_687)', 'remaining time': '23:46:25', 'throughput': '8348.90', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 16:15:53,769 >> {'loss': 0.632, 'grad_norm': 19.269147872924805, 'learning_rate': 9.120062075361155e-07, 'epoch': 0.025494982370490915, 'num_input_tokens_seen': 1874853888, 'completed': '24.25% (894 / 3_687)', 'remaining time': '23:46:46', 'throughput': '8097.06', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 16:16:23,080 >> {'loss': 0.6049, 'grad_norm': 19.273340225219727, 'learning_rate': 9.117662672902584e-07, 'epoch': 0.025766205587198263, 'num_input_tokens_seen': 1876951040, 'completed': '24.27% (895 / 3_687)', 'remaining time': '23:45:36', 'throughput': '8943.66', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 16:16:54,234 >> {'loss': 0.4171, 'grad_norm': 16.550512313842773, 'learning_rate': 9.115260359086757e-07, 'epoch': 0.026037428803905614, 'num_input_tokens_seen': 1879048192, 'completed': '24.30% (896 / 3_687)', 'remaining time': '23:45:20', 'throughput': '8414.27', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 16:17:22,945 >> {'loss': 0.6028, 'grad_norm': 21.415002822875977, 'learning_rate': 9.112855135846964e-07, 'epoch': 0.026308652020612965, 'num_input_tokens_seen': 1881145344, 'completed': '24.33% (897 / 3_687)', 'remaining time': '23:43:54', 'throughput': '9130.67', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 16:17:53,164 >> {'loss': 0.4228, 'grad_norm': 13.791115760803223, 'learning_rate': 9.110447005118836e-07, 'epoch': 0.026579875237320316, 'num_input_tokens_seen': 1883242496, 'completed': '24.36% (898 / 3_687)', 'remaining time': '23:43:12', 'throughput': '8674.63', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 16:18:25,228 >> {'loss': 0.3593, 'grad_norm': 12.052141189575195, 'learning_rate': 9.108035968840348e-07, 'epoch': 0.026851098454027666, 'num_input_tokens_seen': 1885339648, 'completed': '24.38% (899 / 3_687)', 'remaining time': '23:43:22', 'throughput': '8175.78', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 16:18:56,984 >> {'loss': 0.6185, 'grad_norm': 19.59334373474121, 'learning_rate': 9.105622028951806e-07, 'epoch': 0.027122321670735014, 'num_input_tokens_seen': 1887436800, 'completed': '24.41% (900 / 3_687)', 'remaining time': '23:43:23', 'throughput': '8254.87', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 16:19:30,683 >> {'loss': 0.3392, 'grad_norm': 13.629782676696777, 'learning_rate': 9.103205187395861e-07, 'epoch': 0.027393544887442365, 'num_input_tokens_seen': 1889533952, 'completed': '24.44% (901 / 3_687)', 'remaining time': '23:44:17', 'throughput': '7778.90', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 16:20:00,743 >> {'loss': 0.388, 'grad_norm': 14.33342170715332, 'learning_rate': 9.100785446117493e-07, 'epoch': 0.027664768104149716, 'num_input_tokens_seen': 1891631104, 'completed': '24.46% (902 / 3_687)', 'remaining time': '23:43:29', 'throughput': '8720.74', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 16:20:29,097 >> {'loss': 0.5221, 'grad_norm': 17.475322723388672, 'learning_rate': 9.098362807064017e-07, 'epoch': 0.027935991320857067, 'num_input_tokens_seen': 1893728256, 'completed': '24.49% (903 / 3_687)', 'remaining time': '23:41:56', 'throughput': '9245.59', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 16:20:58,559 >> {'loss': 0.3685, 'grad_norm': 15.671577453613281, 'learning_rate': 9.095937272185083e-07, 'epoch': 0.028207214537564414, 'num_input_tokens_seen': 1895825408, 'completed': '24.52% (904 / 3_687)', 'remaining time': '23:40:54', 'throughput': '8897.66', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 16:21:29,067 >> {'loss': 0.4047, 'grad_norm': 16.664928436279297, 'learning_rate': 9.093508843432667e-07, 'epoch': 0.028478437754271765, 'num_input_tokens_seen': 1897922560, 'completed': '24.55% (905 / 3_687)', 'remaining time': '23:40:20', 'throughput': '8592.57', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 16:21:59,558 >> {'loss': 0.6866, 'grad_norm': 23.7731876373291, 'learning_rate': 9.091077522761078e-07, 'epoch': 0.028749660970979116, 'num_input_tokens_seen': 1900019712, 'completed': '24.57% (906 / 3_687)', 'remaining time': '23:39:45', 'throughput': '8597.44', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 16:22:29,158 >> {'loss': 0.6913, 'grad_norm': 18.917259216308594, 'learning_rate': 9.088643312126948e-07, 'epoch': 0.029020884187686467, 'num_input_tokens_seen': 1902116864, 'completed': '24.60% (907 / 3_687)', 'remaining time': '23:38:48', 'throughput': '8856.33', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 16:23:01,117 >> {'loss': 0.4826, 'grad_norm': 16.237077713012695, 'learning_rate': 9.086206213489239e-07, 'epoch': 0.029292107404393815, 'num_input_tokens_seen': 1904214016, 'completed': '24.63% (908 / 3_687)', 'remaining time': '23:38:52', 'throughput': '8202.45', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 16:23:32,115 >> {'loss': 0.6956, 'grad_norm': 19.56369400024414, 'learning_rate': 9.083766228809234e-07, 'epoch': 0.029563330621101166, 'num_input_tokens_seen': 1906311168, 'completed': '24.65% (909 / 3_687)', 'remaining time': '23:38:30', 'throughput': '8456.83', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 16:24:02,245 >> {'loss': 0.4732, 'grad_norm': 14.219721794128418, 'learning_rate': 9.081323360050543e-07, 'epoch': 0.029834553837808517, 'num_input_tokens_seen': 1908408320, 'completed': '24.68% (910 / 3_687)', 'remaining time': '23:37:47', 'throughput': '8700.47', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 16:24:32,457 >> {'loss': 0.4612, 'grad_norm': 17.175357818603516, 'learning_rate': 9.078877609179088e-07, 'epoch': 0.030105777054515868, 'num_input_tokens_seen': 1910505472, 'completed': '24.71% (911 / 3_687)', 'remaining time': '23:37:06', 'throughput': '8676.56', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 16:25:02,164 >> {'loss': 0.3071, 'grad_norm': 12.589146614074707, 'learning_rate': 9.076428978163121e-07, 'epoch': 0.030377000271223215, 'num_input_tokens_seen': 1912602624, 'completed': '24.74% (912 / 3_687)', 'remaining time': '23:36:12', 'throughput': '8824.35', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 16:25:29,515 >> {'loss': 1.031, 'grad_norm': 24.949342727661133, 'learning_rate': 9.073977468973206e-07, 'epoch': 0.030648223487930566, 'num_input_tokens_seen': 1914699776, 'completed': '24.76% (913 / 3_687)', 'remaining time': '23:34:21', 'throughput': '9584.61', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 16:25:58,980 >> {'loss': 0.3996, 'grad_norm': 17.8113956451416, 'learning_rate': 9.071523083582223e-07, 'epoch': 0.030919446704637917, 'num_input_tokens_seen': 1916796928, 'completed': '24.79% (914 / 3_687)', 'remaining time': '23:33:23', 'throughput': '8896.82', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 16:26:27,928 >> {'loss': 0.6838, 'grad_norm': 25.642656326293945, 'learning_rate': 9.06906582396537e-07, 'epoch': 0.031190669921345268, 'num_input_tokens_seen': 1918894080, 'completed': '24.82% (915 / 3_687)', 'remaining time': '23:32:13', 'throughput': '9055.67', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 16:27:00,168 >> {'loss': 0.3932, 'grad_norm': 13.68373966217041, 'learning_rate': 9.066605692100155e-07, 'epoch': 0.031461893138052616, 'num_input_tokens_seen': 1920991232, 'completed': '24.84% (916 / 3_687)', 'remaining time': '23:32:23', 'throughput': '8131.09', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 16:27:31,212 >> {'loss': 0.3473, 'grad_norm': 12.269515037536621, 'learning_rate': 9.064142689966397e-07, 'epoch': 0.03173311635475997, 'num_input_tokens_seen': 1923088384, 'completed': '24.87% (917 / 3_687)', 'remaining time': '23:32:03', 'throughput': '8444.14', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 16:27:58,239 >> {'loss': 0.8176, 'grad_norm': 19.712133407592773, 'learning_rate': 9.061676819546229e-07, 'epoch': 0.03200433957146732, 'num_input_tokens_seen': 1925185536, 'completed': '24.90% (918 / 3_687)', 'remaining time': '23:30:09', 'throughput': '9699.19', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 16:28:29,039 >> {'loss': 0.4822, 'grad_norm': 15.215006828308105, 'learning_rate': 9.059208082824087e-07, 'epoch': 0.032275562788174665, 'num_input_tokens_seen': 1927282688, 'completed': '24.93% (919 / 3_687)', 'remaining time': '23:29:44', 'throughput': '8511.21', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 16:28:59,744 >> {'loss': 0.6293, 'grad_norm': 20.545522689819336, 'learning_rate': 9.05673648178672e-07, 'epoch': 0.03254678600488202, 'num_input_tokens_seen': 1929379840, 'completed': '24.95% (920 / 3_687)', 'remaining time': '23:29:17', 'throughput': '8537.57', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 16:29:31,343 >> {'loss': 0.3147, 'grad_norm': 13.10300350189209, 'learning_rate': 9.054262018423175e-07, 'epoch': 0.03281800922158937, 'num_input_tokens_seen': 1931476992, 'completed': '24.98% (921 / 3_687)', 'remaining time': '23:29:10', 'throughput': '8295.87', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 16:29:59,977 >> {'loss': 0.7481, 'grad_norm': 23.32817268371582, 'learning_rate': 9.051784694724808e-07, 'epoch': 0.03308923243829672, 'num_input_tokens_seen': 1933574144, 'completed': '25.01% (922 / 3_687)', 'remaining time': '23:27:56', 'throughput': '9155.19', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 16:30:31,119 >> {'loss': 0.2809, 'grad_norm': 10.944870948791504, 'learning_rate': 9.049304512685274e-07, 'epoch': 0.03336045565500407, 'num_input_tokens_seen': 1935671296, 'completed': '25.03% (923 / 3_687)', 'remaining time': '23:27:38', 'throughput': '8417.50', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 16:31:00,180 >> {'loss': 0.297, 'grad_norm': 11.271623611450195, 'learning_rate': 9.046821474300527e-07, 'epoch': 0.033631678871711417, 'num_input_tokens_seen': 1937768448, 'completed': '25.06% (924 / 3_687)', 'remaining time': '23:26:35', 'throughput': '9020.76', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 16:31:30,524 >> {'loss': 0.5428, 'grad_norm': 17.19096565246582, 'learning_rate': 9.044335581568827e-07, 'epoch': 0.03390290208841877, 'num_input_tokens_seen': 1939865600, 'completed': '25.09% (925 / 3_687)', 'remaining time': '23:26:00', 'throughput': '8639.02', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 16:31:59,987 >> {'loss': 0.3738, 'grad_norm': 13.172834396362305, 'learning_rate': 9.041846836490723e-07, 'epoch': 0.03417412530512612, 'num_input_tokens_seen': 1941962752, 'completed': '25.12% (926 / 3_687)', 'remaining time': '23:25:05', 'throughput': '8897.23', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 16:32:32,069 >> {'loss': 0.39, 'grad_norm': 13.624527931213379, 'learning_rate': 9.03935524106906e-07, 'epoch': 0.034445348521833466, 'num_input_tokens_seen': 1944059904, 'completed': '25.14% (927 / 3_687)', 'remaining time': '23:25:08', 'throughput': '8171.24', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 16:33:02,317 >> {'loss': 0.2882, 'grad_norm': 11.111466407775879, 'learning_rate': 9.036860797308984e-07, 'epoch': 0.03471657173854082, 'num_input_tokens_seen': 1946157056, 'completed': '25.17% (928 / 3_687)', 'remaining time': '23:24:31', 'throughput': '8666.50', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 16:33:29,961 >> {'loss': 0.4821, 'grad_norm': 18.21550178527832, 'learning_rate': 9.034363507217925e-07, 'epoch': 0.03498779495524817, 'num_input_tokens_seen': 1948254208, 'completed': '25.20% (929 / 3_687)', 'remaining time': '23:22:59', 'throughput': '9482.66', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 16:33:58,943 >> {'loss': 0.6339, 'grad_norm': 25.724552154541016, 'learning_rate': 9.031863372805606e-07, 'epoch': 0.03525901817195552, 'num_input_tokens_seen': 1950351360, 'completed': '25.22% (930 / 3_687)', 'remaining time': '23:21:56', 'throughput': '9045.13', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 16:34:30,358 >> {'loss': 0.2785, 'grad_norm': 22.317489624023438, 'learning_rate': 9.029360396084043e-07, 'epoch': 0.03553024138866287, 'num_input_tokens_seen': 1952448512, 'completed': '25.25% (931 / 3_687)', 'remaining time': '23:21:44', 'throughput': '8344.66', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 16:35:00,449 >> {'loss': 0.502, 'grad_norm': 15.543082237243652, 'learning_rate': 9.026854579067537e-07, 'epoch': 0.03580146460537022, 'num_input_tokens_seen': 1954545664, 'completed': '25.28% (932 / 3_687)', 'remaining time': '23:21:05', 'throughput': '8711.58', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 16:35:31,696 >> {'loss': 0.4526, 'grad_norm': 18.427461624145508, 'learning_rate': 9.024345923772671e-07, 'epoch': 0.03607268782207757, 'num_input_tokens_seen': 1956642816, 'completed': '25.31% (933 / 3_687)', 'remaining time': '23:20:50', 'throughput': '8389.35', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 16:36:00,814 >> {'loss': 0.3389, 'grad_norm': 13.186530113220215, 'learning_rate': 9.021834432218317e-07, 'epoch': 0.03634391103878492, 'num_input_tokens_seen': 1958739968, 'completed': '25.33% (934 / 3_687)', 'remaining time': '23:19:50', 'throughput': '9002.84', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 16:36:31,779 >> {'loss': 0.3434, 'grad_norm': 13.932114601135254, 'learning_rate': 9.019320106425629e-07, 'epoch': 0.03661513425549227, 'num_input_tokens_seen': 1960837120, 'completed': '25.36% (935 / 3_687)', 'remaining time': '23:19:29', 'throughput': '8465.92', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 16:37:01,500 >> {'loss': 0.2254, 'grad_norm': 12.41275691986084, 'learning_rate': 9.016802948418038e-07, 'epoch': 0.03688635747219962, 'num_input_tokens_seen': 1962934272, 'completed': '25.39% (936 / 3_687)', 'remaining time': '23:18:42', 'throughput': '8820.00', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 16:37:29,385 >> {'loss': 0.5891, 'grad_norm': 18.870506286621094, 'learning_rate': 9.014282960221257e-07, 'epoch': 0.03715758068890697, 'num_input_tokens_seen': 1965031424, 'completed': '25.41% (937 / 3_687)', 'remaining time': '23:17:19', 'throughput': '9400.87', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 16:38:00,095 >> {'loss': 0.4418, 'grad_norm': 15.504498481750488, 'learning_rate': 9.01176014386328e-07, 'epoch': 0.03742880390561432, 'num_input_tokens_seen': 1967128576, 'completed': '25.44% (938 / 3_687)', 'remaining time': '23:16:53', 'throughput': '8536.25', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 16:38:28,922 >> {'loss': 0.6721, 'grad_norm': 19.771808624267578, 'learning_rate': 9.009234501374371e-07, 'epoch': 0.03770002712232167, 'num_input_tokens_seen': 1969225728, 'completed': '25.47% (939 / 3_687)', 'remaining time': '23:15:50', 'throughput': '9093.59', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 16:39:01,111 >> {'loss': 0.5628, 'grad_norm': 23.42245101928711, 'learning_rate': 9.006706034787071e-07, 'epoch': 0.03797125033902902, 'num_input_tokens_seen': 1971322880, 'completed': '25.49% (940 / 3_687)', 'remaining time': '23:15:53', 'throughput': '8144.06', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 16:39:29,356 >> {'loss': 0.6278, 'grad_norm': 18.261873245239258, 'learning_rate': 9.004174746136196e-07, 'epoch': 0.03824247355573637, 'num_input_tokens_seen': 1973420032, 'completed': '25.52% (941 / 3_687)', 'remaining time': '23:14:39', 'throughput': '9281.07', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 16:40:03,845 >> {'loss': 0.4231, 'grad_norm': 15.880273818969727, 'learning_rate': 9.001640637458829e-07, 'epoch': 0.03851369677244372, 'num_input_tokens_seen': 1975517184, 'completed': '25.55% (942 / 3_687)', 'remaining time': '23:15:26', 'throughput': '7600.61', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 16:40:32,662 >> {'loss': 0.8923, 'grad_norm': 26.52276611328125, 'learning_rate': 8.999103710794323e-07, 'epoch': 0.038784919989151075, 'num_input_tokens_seen': 1977614336, 'completed': '25.58% (943 / 3_687)', 'remaining time': '23:14:23', 'throughput': '9097.13', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 16:41:02,778 >> {'loss': 0.3194, 'grad_norm': 12.265298843383789, 'learning_rate': 8.996563968184302e-07, 'epoch': 0.03905614320585842, 'num_input_tokens_seen': 1979711488, 'completed': '25.60% (944 / 3_687)', 'remaining time': '23:13:46', 'throughput': '8704.22', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 16:41:33,474 >> {'loss': 0.7034, 'grad_norm': 20.407365798950195, 'learning_rate': 8.994021411672653e-07, 'epoch': 0.03932736642256577, 'num_input_tokens_seen': 1981808640, 'completed': '25.63% (945 / 3_687)', 'remaining time': '23:13:19', 'throughput': '8540.12', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 16:42:02,716 >> {'loss': 0.5223, 'grad_norm': 22.567445755004883, 'learning_rate': 8.99147604330553e-07, 'epoch': 0.039598589639273124, 'num_input_tokens_seen': 1983905792, 'completed': '25.66% (946 / 3_687)', 'remaining time': '23:12:25', 'throughput': '8964.64', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 16:42:35,359 >> {'loss': 0.5478, 'grad_norm': 30.489513397216797, 'learning_rate': 8.988927865131347e-07, 'epoch': 0.03986981285598047, 'num_input_tokens_seen': 1986002944, 'completed': '25.68% (947 / 3_687)', 'remaining time': '23:12:35', 'throughput': '8030.53', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 16:43:07,617 >> {'loss': 0.5236, 'grad_norm': 18.10291862487793, 'learning_rate': 8.986376879200783e-07, 'epoch': 0.04014103607268782, 'num_input_tokens_seen': 1988100096, 'completed': '25.71% (948 / 3_687)', 'remaining time': '23:12:37', 'throughput': '8126.59', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 16:43:38,583 >> {'loss': 0.2878, 'grad_norm': 13.505256652832031, 'learning_rate': 8.983823087566772e-07, 'epoch': 0.040412259289395173, 'num_input_tokens_seen': 1990197248, 'completed': '25.74% (949 / 3_687)', 'remaining time': '23:12:15', 'throughput': '8465.67', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 16:44:09,662 >> {'loss': 0.5118, 'grad_norm': 16.355928421020508, 'learning_rate': 8.981266492284511e-07, 'epoch': 0.04068348250610252, 'num_input_tokens_seen': 1992294400, 'completed': '25.77% (950 / 3_687)', 'remaining time': '23:11:55', 'throughput': '8434.68', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 16:44:41,733 >> {'loss': 0.6593, 'grad_norm': 20.954561233520508, 'learning_rate': 8.978707095411446e-07, 'epoch': 0.040954705722809875, 'num_input_tokens_seen': 1994391552, 'completed': '25.79% (951 / 3_687)', 'remaining time': '23:11:53', 'throughput': '8174.05', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 16:45:10,174 >> {'loss': 1.0425, 'grad_norm': 28.557451248168945, 'learning_rate': 8.976144899007288e-07, 'epoch': 0.04122592893951722, 'num_input_tokens_seen': 1996488704, 'completed': '25.82% (952 / 3_687)', 'remaining time': '23:10:45', 'throughput': '9216.91', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 16:45:40,680 >> {'loss': 0.4375, 'grad_norm': 13.614970207214355, 'learning_rate': 8.973579905133991e-07, 'epoch': 0.04149715215622457, 'num_input_tokens_seen': 1998585856, 'completed': '25.85% (953 / 3_687)', 'remaining time': '23:10:14', 'throughput': '8593.08', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 16:46:11,488 >> {'loss': 0.2845, 'grad_norm': 10.78772258758545, 'learning_rate': 8.971012115855766e-07, 'epoch': 0.041768375372931925, 'num_input_tokens_seen': 2000683008, 'completed': '25.87% (954 / 3_687)', 'remaining time': '23:09:49', 'throughput': '8508.93', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 16:46:41,249 >> {'loss': 0.4906, 'grad_norm': 17.9395694732666, 'learning_rate': 8.968441533239073e-07, 'epoch': 0.04203959858963927, 'num_input_tokens_seen': 2002780160, 'completed': '25.90% (955 / 3_687)', 'remaining time': '23:09:05', 'throughput': '8808.66', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 16:47:12,945 >> {'loss': 0.4028, 'grad_norm': 13.34605884552002, 'learning_rate': 8.965868159352616e-07, 'epoch': 0.04231082180634662, 'num_input_tokens_seen': 2004877312, 'completed': '25.93% (956 / 3_687)', 'remaining time': '23:08:55', 'throughput': '8270.42', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 16:47:43,577 >> {'loss': 0.3782, 'grad_norm': 16.155813217163086, 'learning_rate': 8.963291996267354e-07, 'epoch': 0.042582045023053974, 'num_input_tokens_seen': 2006974464, 'completed': '25.96% (957 / 3_687)', 'remaining time': '23:08:27', 'throughput': '8557.91', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 16:48:16,018 >> {'loss': 0.3243, 'grad_norm': 11.215263366699219, 'learning_rate': 8.960713046056478e-07, 'epoch': 0.04285326823976132, 'num_input_tokens_seen': 2009071616, 'completed': '25.98% (958 / 3_687)', 'remaining time': '23:08:30', 'throughput': '8080.63', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 16:48:44,323 >> {'loss': 0.4452, 'grad_norm': 13.548431396484375, 'learning_rate': 8.958131310795434e-07, 'epoch': 0.043124491456468676, 'num_input_tokens_seen': 2011168768, 'completed': '26.01% (959 / 3_687)', 'remaining time': '23:07:21', 'throughput': '9261.32', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 16:49:14,761 >> {'loss': 0.4006, 'grad_norm': 13.86419677734375, 'learning_rate': 8.955546792561902e-07, 'epoch': 0.043395714673176024, 'num_input_tokens_seen': 2013265920, 'completed': '26.04% (960 / 3_687)', 'remaining time': '23:06:49', 'throughput': '8612.39', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 16:49:44,699 >> {'loss': 0.3906, 'grad_norm': 18.16084098815918, 'learning_rate': 8.952959493435806e-07, 'epoch': 0.04366693788988337, 'num_input_tokens_seen': 2015363072, 'completed': '26.06% (961 / 3_687)', 'remaining time': '23:06:09', 'throughput': '8756.12', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 16:50:13,856 >> {'loss': 0.5608, 'grad_norm': 18.875059127807617, 'learning_rate': 8.950369415499304e-07, 'epoch': 0.043938161106590726, 'num_input_tokens_seen': 2017460224, 'completed': '26.09% (962 / 3_687)', 'remaining time': '23:05:16', 'throughput': '8990.89', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 16:50:48,212 >> {'loss': 0.7657, 'grad_norm': 22.01349639892578, 'learning_rate': 8.947776560836793e-07, 'epoch': 0.04420938432329807, 'num_input_tokens_seen': 2019557376, 'completed': '26.12% (963 / 3_687)', 'remaining time': '23:05:50', 'throughput': '7630.21', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 16:51:19,253 >> {'loss': 0.4577, 'grad_norm': 17.236188888549805, 'learning_rate': 8.945180931534902e-07, 'epoch': 0.04448060754000543, 'num_input_tokens_seen': 2021654528, 'completed': '26.15% (964 / 3_687)', 'remaining time': '23:05:28', 'throughput': '8445.05', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 16:51:51,635 >> {'loss': 0.4122, 'grad_norm': 15.46218204498291, 'learning_rate': 8.942582529682496e-07, 'epoch': 0.044751830756712775, 'num_input_tokens_seen': 2023751680, 'completed': '26.17% (965 / 3_687)', 'remaining time': '23:05:28', 'throughput': '8095.43', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 16:52:21,810 >> {'loss': 0.3415, 'grad_norm': 14.264276504516602, 'learning_rate': 8.939981357370672e-07, 'epoch': 0.04502305397342012, 'num_input_tokens_seen': 2025848832, 'completed': '26.20% (966 / 3_687)', 'remaining time': '23:04:51', 'throughput': '8687.49', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 16:52:52,625 >> {'loss': 0.4217, 'grad_norm': 17.676618576049805, 'learning_rate': 8.937377416692752e-07, 'epoch': 0.04529427719012748, 'num_input_tokens_seen': 2027945984, 'completed': '26.23% (967 / 3_687)', 'remaining time': '23:04:25', 'throughput': '8506.96', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 16:53:22,579 >> {'loss': 0.4929, 'grad_norm': 16.618602752685547, 'learning_rate': 8.934770709744289e-07, 'epoch': 0.045565500406834825, 'num_input_tokens_seen': 2030043136, 'completed': '26.25% (968 / 3_687)', 'remaining time': '23:03:45', 'throughput': '8751.44', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 16:53:55,019 >> {'loss': 0.3282, 'grad_norm': 13.402697563171387, 'learning_rate': 8.93216123862306e-07, 'epoch': 0.04583672362354217, 'num_input_tokens_seen': 2032140288, 'completed': '26.28% (969 / 3_687)', 'remaining time': '23:03:45', 'throughput': '8081.06', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 16:54:22,026 >> {'loss': 1.1138, 'grad_norm': 24.799013137817383, 'learning_rate': 8.929549005429071e-07, 'epoch': 0.04610794684024953, 'num_input_tokens_seen': 2034237440, 'completed': '26.31% (970 / 3_687)', 'remaining time': '23:02:18', 'throughput': '9706.24', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 16:54:52,147 >> {'loss': 0.4161, 'grad_norm': 17.392616271972656, 'learning_rate': 8.926934012264546e-07, 'epoch': 0.046379170056956874, 'num_input_tokens_seen': 2036334592, 'completed': '26.34% (971 / 3_687)', 'remaining time': '23:01:41', 'throughput': '8703.17', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 16:55:23,872 >> {'loss': 0.567, 'grad_norm': 18.09572410583496, 'learning_rate': 8.924316261233933e-07, 'epoch': 0.04665039327366423, 'num_input_tokens_seen': 2038431744, 'completed': '26.36% (972 / 3_687)', 'remaining time': '23:01:30', 'throughput': '8262.89', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 16:55:55,006 >> {'loss': 0.3801, 'grad_norm': 14.815757751464844, 'learning_rate': 8.921695754443898e-07, 'epoch': 0.046921616490371576, 'num_input_tokens_seen': 2040528896, 'completed': '26.39% (973 / 3_687)', 'remaining time': '23:01:09', 'throughput': '8419.87', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 16:56:23,994 >> {'loss': 0.4794, 'grad_norm': 16.38593864440918, 'learning_rate': 8.919072494003325e-07, 'epoch': 0.047192839707078924, 'num_input_tokens_seen': 2042626048, 'completed': '26.42% (974 / 3_687)', 'remaining time': '23:00:14', 'throughput': '9043.19', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 16:56:57,223 >> {'loss': 0.394, 'grad_norm': 13.091635704040527, 'learning_rate': 8.916446482023313e-07, 'epoch': 0.04746406292378628, 'num_input_tokens_seen': 2044723200, 'completed': '26.44% (975 / 3_687)', 'remaining time': '23:00:25', 'throughput': '7889.03', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 16:57:26,618 >> {'loss': 0.5268, 'grad_norm': 17.225982666015625, 'learning_rate': 8.913817720617178e-07, 'epoch': 0.047735286140493625, 'num_input_tokens_seen': 2046820352, 'completed': '26.47% (976 / 3_687)', 'remaining time': '22:59:37', 'throughput': '8918.13', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 16:57:59,009 >> {'loss': 0.3478, 'grad_norm': 12.275423049926758, 'learning_rate': 8.911186211900448e-07, 'epoch': 0.04800650935720097, 'num_input_tokens_seen': 2048917504, 'completed': '26.50% (977 / 3_687)', 'remaining time': '22:59:35', 'throughput': '8093.09', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 16:58:31,641 >> {'loss': 0.619, 'grad_norm': 19.183958053588867, 'learning_rate': 8.908551957990858e-07, 'epoch': 0.04827773257390833, 'num_input_tokens_seen': 2051014656, 'completed': '26.53% (978 / 3_687)', 'remaining time': '22:59:36', 'throughput': '8033.18', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 16:59:00,661 >> {'loss': 0.6647, 'grad_norm': 22.74364471435547, 'learning_rate': 8.90591496100836e-07, 'epoch': 0.048548955790615675, 'num_input_tokens_seen': 2053111808, 'completed': '26.55% (979 / 3_687)', 'remaining time': '22:58:42', 'throughput': '9033.24', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 16:59:29,996 >> {'loss': 0.5442, 'grad_norm': 20.398710250854492, 'learning_rate': 8.903275223075104e-07, 'epoch': 0.04882017900732303, 'num_input_tokens_seen': 2055208960, 'completed': '26.58% (980 / 3_687)', 'remaining time': '22:57:54', 'throughput': '8936.43', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 16:59:58,409 >> {'loss': 0.683, 'grad_norm': 19.808082580566406, 'learning_rate': 8.900632746315454e-07, 'epoch': 0.04909140222403038, 'num_input_tokens_seen': 2057306112, 'completed': '26.61% (981 / 3_687)', 'remaining time': '22:56:51', 'throughput': '9226.10', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 17:00:27,001 >> {'loss': 1.0303, 'grad_norm': 26.348705291748047, 'learning_rate': 8.897987532855972e-07, 'epoch': 0.049362625440737724, 'num_input_tokens_seen': 2059403264, 'completed': '26.63% (982 / 3_687)', 'remaining time': '22:55:52', 'throughput': '9168.46', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 17:01:00,917 >> {'loss': 0.5595, 'grad_norm': 21.460338592529297, 'learning_rate': 8.895339584825429e-07, 'epoch': 0.04963384865744508, 'num_input_tokens_seen': 2061500416, 'completed': '26.66% (983 / 3_687)', 'remaining time': '22:56:12', 'throughput': '7729.12', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 17:01:34,415 >> {'loss': 0.4599, 'grad_norm': 14.994651794433594, 'learning_rate': 8.892688904354787e-07, 'epoch': 0.049905071874152426, 'num_input_tokens_seen': 2063597568, 'completed': '26.69% (984 / 3_687)', 'remaining time': '22:56:25', 'throughput': '7825.80', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 17:02:03,784 >> {'loss': 0.373, 'grad_norm': 12.65827751159668, 'learning_rate': 8.890035493577219e-07, 'epoch': 0.05017629509085978, 'num_input_tokens_seen': 2065694720, 'completed': '26.72% (985 / 3_687)', 'remaining time': '22:55:37', 'throughput': '8925.71', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 17:02:33,403 >> {'loss': 0.9004, 'grad_norm': 22.335948944091797, 'learning_rate': 8.887379354628085e-07, 'epoch': 0.05044751830756713, 'num_input_tokens_seen': 2067791872, 'completed': '26.74% (986 / 3_687)', 'remaining time': '22:54:53', 'throughput': '8850.65', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 17:03:02,335 >> {'loss': 0.3551, 'grad_norm': 13.694454193115234, 'learning_rate': 8.884720489644945e-07, 'epoch': 0.050718741524274476, 'num_input_tokens_seen': 2069889024, 'completed': '26.77% (987 / 3_687)', 'remaining time': '22:53:59', 'throughput': '9060.54', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 17:03:29,658 >> {'loss': 0.735, 'grad_norm': 20.320920944213867, 'learning_rate': 8.882058900767555e-07, 'epoch': 0.05098996474098183, 'num_input_tokens_seen': 2071986176, 'completed': '26.80% (988 / 3_687)', 'remaining time': '22:52:42', 'throughput': '9594.43', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 17:04:04,966 >> {'loss': 0.5857, 'grad_norm': 19.12615966796875, 'learning_rate': 8.879394590137857e-07, 'epoch': 0.05126118795768918, 'num_input_tokens_seen': 2074083328, 'completed': '26.82% (989 / 3_687)', 'remaining time': '22:53:20', 'throughput': '7424.51', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 17:04:37,521 >> {'loss': 0.3749, 'grad_norm': 16.400854110717773, 'learning_rate': 8.876727559899989e-07, 'epoch': 0.051532411174396525, 'num_input_tokens_seen': 2076180480, 'completed': '26.85% (990 / 3_687)', 'remaining time': '22:53:18', 'throughput': '8052.30', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 17:05:04,342 >> {'loss': 0.9273, 'grad_norm': 24.33226203918457, 'learning_rate': 8.874057812200274e-07, 'epoch': 0.05180363439110388, 'num_input_tokens_seen': 2078277632, 'completed': '26.88% (991 / 3_687)', 'remaining time': '22:51:55', 'throughput': '9773.94', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 17:05:35,182 >> {'loss': 0.6876, 'grad_norm': 17.637216567993164, 'learning_rate': 8.871385349187225e-07, 'epoch': 0.05207485760781123, 'num_input_tokens_seen': 2080374784, 'completed': '26.91% (992 / 3_687)', 'remaining time': '22:51:29', 'throughput': '8500.08', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 17:06:03,983 >> {'loss': 0.7319, 'grad_norm': 22.15195083618164, 'learning_rate': 8.868710173011538e-07, 'epoch': 0.05234608082451858, 'num_input_tokens_seen': 2082471936, 'completed': '26.93% (993 / 3_687)', 'remaining time': '22:50:34', 'throughput': '9101.89', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 17:06:34,219 >> {'loss': 0.4949, 'grad_norm': 16.152517318725586, 'learning_rate': 8.866032285826091e-07, 'epoch': 0.05261730404122593, 'num_input_tokens_seen': 2084569088, 'completed': '26.96% (994 / 3_687)', 'remaining time': '22:50:00', 'throughput': '8670.02', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 17:07:06,973 >> {'loss': 0.3432, 'grad_norm': 13.20339584350586, 'learning_rate': 8.86335168978595e-07, 'epoch': 0.05288852725793328, 'num_input_tokens_seen': 2086666240, 'completed': '26.99% (995 / 3_687)', 'remaining time': '22:50:00', 'throughput': '8003.29', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 17:07:39,120 >> {'loss': 0.2759, 'grad_norm': 12.26603889465332, 'learning_rate': 8.860668387048353e-07, 'epoch': 0.05315975047464063, 'num_input_tokens_seen': 2088763392, 'completed': '27.01% (996 / 3_687)', 'remaining time': '22:49:51', 'throughput': '8154.48', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 17:08:09,471 >> {'loss': 0.3028, 'grad_norm': 12.985957145690918, 'learning_rate': 8.85798237977272e-07, 'epoch': 0.05343097369134798, 'num_input_tokens_seen': 2090860544, 'completed': '27.04% (997 / 3_687)', 'remaining time': '22:49:18', 'throughput': '8637.28', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 17:08:42,451 >> {'loss': 0.63, 'grad_norm': 17.120885848999023, 'learning_rate': 8.85529367012065e-07, 'epoch': 0.05370219690805533, 'num_input_tokens_seen': 2092957696, 'completed': '27.07% (998 / 3_687)', 'remaining time': '22:49:21', 'throughput': '7948.65', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 17:09:13,732 >> {'loss': 0.4986, 'grad_norm': 17.12887191772461, 'learning_rate': 8.852602260255911e-07, 'epoch': 0.05397342012476268, 'num_input_tokens_seen': 2095054848, 'completed': '27.10% (999 / 3_687)', 'remaining time': '22:49:00', 'throughput': '8380.18', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 17:09:45,155 >> {'loss': 0.7428, 'grad_norm': 19.522432327270508, 'learning_rate': 8.849908152344451e-07, 'epoch': 0.05424464334147003, 'num_input_tokens_seen': 2097152000, 'completed': '27.12% (1_000 / 3_687)', 'remaining time': '22:48:41', 'throughput': '8342.42', 'gpu_mem_free': '5581MB'} +/scratch3/workspace/ctpham_umass_edu-ft/envs/prolong-final/lib/python3.10/site-packages/torch/distributed/fsdp/fully_sharded_data_parallel.py:689: FutureWarning: FSDP.state_dict_type() and FSDP.set_state_dict_type() are being deprecated. Please use APIs, get_state_dict() and set_state_dict(), which can support different parallelisms, FSDP1, FSDP2, DDP. API doc: https://pytorch.org/docs/stable/distributed.checkpoint.html#torch.distributed.checkpoint.state_dict.get_state_dict .Tutorial: https://pytorch.org/tutorials/recipes/distributed_checkpoint_recipe.html . + warnings.warn( +[INFO|trainer.py:3503] 2024-12-25 17:10:09,029 >> Saving model checkpoint to /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/checkpoint-1000 +[INFO|configuration_utils.py:472] 2024-12-25 17:10:09,034 >> Configuration saved in /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/checkpoint-1000/config.json +[INFO|configuration_utils.py:807] 2024-12-25 17:10:09,035 >> Configuration saved in /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/checkpoint-1000/generation_config.json +[INFO|modeling_utils.py:2807] 2024-12-25 17:11:05,268 >> The model is bigger than the maximum size per checkpoint (5GB) and is going to be split in 7 checkpoint shards. You can find where each parameters has been saved in the index located at /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/checkpoint-1000/model.safetensors.index.json. +[INFO|tokenization_utils_base.py:2684] 2024-12-25 17:11:05,273 >> tokenizer config file saved in /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/checkpoint-1000/tokenizer_config.json +[INFO|tokenization_utils_base.py:2693] 2024-12-25 17:11:05,274 >> Special tokens file saved in /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/checkpoint-1000/special_tokens_map.json +/scratch3/workspace/ctpham_umass_edu-ft/envs/prolong-final/lib/python3.10/site-packages/torch/distributed/fsdp/fully_sharded_data_parallel.py:689: FutureWarning: FSDP.state_dict_type() and FSDP.set_state_dict_type() are being deprecated. Please use APIs, get_state_dict() and set_state_dict(), which can support different parallelisms, FSDP1, FSDP2, DDP. API doc: https://pytorch.org/docs/stable/distributed.checkpoint.html#torch.distributed.checkpoint.state_dict.get_state_dict .Tutorial: https://pytorch.org/tutorials/recipes/distributed_checkpoint_recipe.html . + warnings.warn( +[WARNING|trainer.py:868] 2024-12-25 17:14:59,159 >> Save streaming dataset state: {'epoch': 0, 'sample_in_epoch': 2000, 'num_canonical_nodes': 1, 'shuffle_seed': 42, 'initial_physical_nodes': 1} +12/25/2024 17:14:59 - WARNING - streaming.base.dataset - Because `shuffle_block_size` was not specified, it will default to max(4_000_000 // num_canonical_nodes, 1 << 18) if num_canonical_nodes is not None, otherwise 262144. Prior to Streaming v0.7.0, `shuffle_block_size` defaulted to 262144. +/scratch3/workspace/ctpham_umass_edu-ft/envs/prolong-final/lib/python3.10/site-packages/torch/utils/checkpoint.py:1399: FutureWarning: `torch.cpu.amp.autocast(args...)` is deprecated. Please use `torch.amp.autocast('cpu', args...)` instead. + with device_autocast_ctx, torch.cpu.amp.autocast(**cpu_autocast_kwargs), recompute_context: # type: ignore[attr-defined] +[INFO|trainer.py:175] 2024-12-25 17:15:29,462 >> {'loss': 0.3491, 'grad_norm': 19.57962417602539, 'learning_rate': 8.847211348554382e-07, 'epoch': 0.05451586655817738, 'num_input_tokens_seen': 2099249152, 'completed': '27.15% (1_001 / 3_687)', 'remaining time': '23:58:03', 'throughput': '761.37', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 17:16:01,123 >> {'loss': 0.4238, 'grad_norm': 15.201761245727539, 'learning_rate': 8.844511851055991e-07, 'epoch': 0.05478708977488473, 'num_input_tokens_seen': 2101346304, 'completed': '27.18% (1_002 / 3_687)', 'remaining time': '23:57:25', 'throughput': '8279.79', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 17:16:32,013 >> {'loss': 0.4377, 'grad_norm': 15.727082252502441, 'learning_rate': 8.841809662021731e-07, 'epoch': 0.05505831299159208, 'num_input_tokens_seen': 2103443456, 'completed': '27.20% (1_003 / 3_687)', 'remaining time': '23:56:37', 'throughput': '8486.28', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 17:17:02,930 >> {'loss': 0.7603, 'grad_norm': 22.993534088134766, 'learning_rate': 8.839104783626219e-07, 'epoch': 0.05532953620829943, 'num_input_tokens_seen': 2105540608, 'completed': '27.23% (1_004 / 3_687)', 'remaining time': '23:55:49', 'throughput': '8479.07', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 17:17:32,870 >> {'loss': 0.3576, 'grad_norm': 13.314918518066406, 'learning_rate': 8.836397218046239e-07, 'epoch': 0.05560075942500678, 'num_input_tokens_seen': 2107637760, 'completed': '27.26% (1_005 / 3_687)', 'remaining time': '23:54:48', 'throughput': '8755.50', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 17:18:01,940 >> {'loss': 0.4315, 'grad_norm': 14.432062149047852, 'learning_rate': 8.83368696746074e-07, 'epoch': 0.055871982641714134, 'num_input_tokens_seen': 2109734912, 'completed': '27.29% (1_006 / 3_687)', 'remaining time': '23:53:37', 'throughput': '9017.71', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 17:18:30,956 >> {'loss': 0.7692, 'grad_norm': 20.760156631469727, 'learning_rate': 8.830974034050824e-07, 'epoch': 0.05614320585842148, 'num_input_tokens_seen': 2111832064, 'completed': '27.31% (1_007 / 3_687)', 'remaining time': '23:52:25', 'throughput': '9034.47', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 17:18:59,818 >> {'loss': 0.283, 'grad_norm': 10.294124603271484, 'learning_rate': 8.828258419999759e-07, 'epoch': 0.05641442907512883, 'num_input_tokens_seen': 2113929216, 'completed': '27.34% (1_008 / 3_687)', 'remaining time': '23:51:12', 'throughput': '9082.63', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 17:19:31,762 >> {'loss': 0.2858, 'grad_norm': 10.813933372497559, 'learning_rate': 8.825540127492965e-07, 'epoch': 0.05668565229183618, 'num_input_tokens_seen': 2116026368, 'completed': '27.37% (1_009 / 3_687)', 'remaining time': '23:50:38', 'throughput': '8206.35', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 17:20:03,199 >> {'loss': 0.717, 'grad_norm': 23.46965217590332, 'learning_rate': 8.822819158718026e-07, 'epoch': 0.05695687550854353, 'num_input_tokens_seen': 2118123520, 'completed': '27.39% (1_010 / 3_687)', 'remaining time': '23:49:58', 'throughput': '8338.77', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 17:20:33,501 >> {'loss': 0.5001, 'grad_norm': 19.347179412841797, 'learning_rate': 8.820095515864669e-07, 'epoch': 0.05722809872525088, 'num_input_tokens_seen': 2120220672, 'completed': '27.42% (1_011 / 3_687)', 'remaining time': '23:49:04', 'throughput': '8650.98', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 17:21:03,570 >> {'loss': 0.5409, 'grad_norm': 18.49291229248047, 'learning_rate': 8.81736920112478e-07, 'epoch': 0.05749932194195823, 'num_input_tokens_seen': 2122317824, 'completed': '27.45% (1_012 / 3_687)', 'remaining time': '23:48:07', 'throughput': '8718.03', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 17:21:38,394 >> {'loss': 0.4008, 'grad_norm': 14.770519256591797, 'learning_rate': 8.814640216692391e-07, 'epoch': 0.05777054515866558, 'num_input_tokens_seen': 2124414976, 'completed': '27.47% (1_013 / 3_687)', 'remaining time': '23:48:10', 'throughput': '7527.69', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 17:22:05,927 >> {'loss': 0.8029, 'grad_norm': 22.392454147338867, 'learning_rate': 8.81190856476369e-07, 'epoch': 0.058041768375372935, 'num_input_tokens_seen': 2126512128, 'completed': '27.50% (1_014 / 3_687)', 'remaining time': '23:46:42', 'throughput': '9521.16', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 17:22:35,257 >> {'loss': 0.5948, 'grad_norm': 19.39711570739746, 'learning_rate': 8.809174247537003e-07, 'epoch': 0.05831299159208028, 'num_input_tokens_seen': 2128609280, 'completed': '27.53% (1_015 / 3_687)', 'remaining time': '23:45:36', 'throughput': '8937.85', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 17:23:07,442 >> {'loss': 0.2487, 'grad_norm': 11.22283935546875, 'learning_rate': 8.806437267212805e-07, 'epoch': 0.05858421480878763, 'num_input_tokens_seen': 2130706432, 'completed': '27.56% (1_016 / 3_687)', 'remaining time': '23:45:06', 'throughput': '8144.88', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 17:23:41,141 >> {'loss': 0.8866, 'grad_norm': 23.879262924194336, 'learning_rate': 8.803697625993713e-07, 'epoch': 0.058855438025494984, 'num_input_tokens_seen': 2132803584, 'completed': '27.58% (1_017 / 3_687)', 'remaining time': '23:44:55', 'throughput': '7778.92', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 17:24:10,954 >> {'loss': 0.6154, 'grad_norm': 17.010164260864258, 'learning_rate': 8.800955326084487e-07, 'epoch': 0.05912666124220233, 'num_input_tokens_seen': 2134900736, 'completed': '27.61% (1_018 / 3_687)', 'remaining time': '23:43:56', 'throughput': '8792.97', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 17:24:41,449 >> {'loss': 0.2381, 'grad_norm': 12.31965160369873, 'learning_rate': 8.798210369692025e-07, 'epoch': 0.059397884458909686, 'num_input_tokens_seen': 2136997888, 'completed': '27.64% (1_019 / 3_687)', 'remaining time': '23:43:05', 'throughput': '8596.30', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 17:25:12,178 >> {'loss': 0.4989, 'grad_norm': 14.956830978393555, 'learning_rate': 8.795462759025364e-07, 'epoch': 0.059669107675617034, 'num_input_tokens_seen': 2139095040, 'completed': '27.66% (1_020 / 3_687)', 'remaining time': '23:42:18', 'throughput': '8530.81', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 17:25:44,706 >> {'loss': 0.3168, 'grad_norm': 11.59531021118164, 'learning_rate': 8.792712496295677e-07, 'epoch': 0.05994033089232438, 'num_input_tokens_seen': 2141192192, 'completed': '27.69% (1_021 / 3_687)', 'remaining time': '23:41:52', 'throughput': '8058.92', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 17:26:15,354 >> {'loss': 0.4643, 'grad_norm': 15.695453643798828, 'learning_rate': 8.789959583716268e-07, 'epoch': 0.060211554109031735, 'num_input_tokens_seen': 2143289344, 'completed': '27.72% (1_022 / 3_687)', 'remaining time': '23:41:04', 'throughput': '8553.66', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 17:26:45,292 >> {'loss': 0.2886, 'grad_norm': 12.228116989135742, 'learning_rate': 8.787204023502579e-07, 'epoch': 0.06048277732573908, 'num_input_tokens_seen': 2145386496, 'completed': '27.75% (1_023 / 3_687)', 'remaining time': '23:40:08', 'throughput': '8756.13', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 17:27:12,539 >> {'loss': 0.4706, 'grad_norm': 15.869124412536621, 'learning_rate': 8.78444581787218e-07, 'epoch': 0.06075400054244643, 'num_input_tokens_seen': 2147483648, 'completed': '27.77% (1_024 / 3_687)', 'remaining time': '23:38:39', 'throughput': '9621.25', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 17:27:44,870 >> {'loss': 0.4961, 'grad_norm': 17.77741241455078, 'learning_rate': 8.781684969044769e-07, 'epoch': 0.061025223759153785, 'num_input_tokens_seen': 2149580800, 'completed': '27.80% (1_025 / 3_687)', 'remaining time': '23:38:12', 'throughput': '8107.97', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 17:28:16,783 >> {'loss': 0.5621, 'grad_norm': 18.270593643188477, 'learning_rate': 8.778921479242173e-07, 'epoch': 0.06129644697586113, 'num_input_tokens_seen': 2151677952, 'completed': '27.83% (1_026 / 3_687)', 'remaining time': '23:37:39', 'throughput': '8214.23', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 17:28:47,320 >> {'loss': 0.2092, 'grad_norm': 10.54736614227295, 'learning_rate': 8.776155350688342e-07, 'epoch': 0.06156767019256849, 'num_input_tokens_seen': 2153775104, 'completed': '27.85% (1_027 / 3_687)', 'remaining time': '23:36:50', 'throughput': '8584.49', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 17:29:17,452 >> {'loss': 0.2644, 'grad_norm': 14.527905464172363, 'learning_rate': 8.773386585609352e-07, 'epoch': 0.061838893409275834, 'num_input_tokens_seen': 2155872256, 'completed': '27.88% (1_028 / 3_687)', 'remaining time': '23:35:57', 'throughput': '8700.03', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 17:29:46,966 >> {'loss': 0.4327, 'grad_norm': 14.493075370788574, 'learning_rate': 8.770615186233398e-07, 'epoch': 0.06211011662598318, 'num_input_tokens_seen': 2157969408, 'completed': '27.91% (1_029 / 3_687)', 'remaining time': '23:34:57', 'throughput': '8881.99', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 17:30:17,553 >> {'loss': 0.3887, 'grad_norm': 15.283905982971191, 'learning_rate': 8.7678411547908e-07, 'epoch': 0.062381339842690536, 'num_input_tokens_seen': 2160066560, 'completed': '27.94% (1_030 / 3_687)', 'remaining time': '23:34:09', 'throughput': '8570.46', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 17:30:46,609 >> {'loss': 0.8746, 'grad_norm': 22.41847801208496, 'learning_rate': 8.76506449351399e-07, 'epoch': 0.06265256305939788, 'num_input_tokens_seen': 2162163712, 'completed': '27.96% (1_031 / 3_687)', 'remaining time': '23:33:04', 'throughput': '9021.79', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 17:31:19,114 >> {'loss': 0.3222, 'grad_norm': 12.715897560119629, 'learning_rate': 8.762285204637522e-07, 'epoch': 0.06292378627610523, 'num_input_tokens_seen': 2164260864, 'completed': '27.99% (1_032 / 3_687)', 'remaining time': '23:32:39', 'throughput': '8064.76', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 17:31:47,107 >> {'loss': 0.9287, 'grad_norm': 25.04868507385254, 'learning_rate': 8.75950329039806e-07, 'epoch': 0.06319500949281258, 'num_input_tokens_seen': 2166358016, 'completed': '28.02% (1_033 / 3_687)', 'remaining time': '23:31:22', 'throughput': '9364.55', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 17:32:18,344 >> {'loss': 0.4544, 'grad_norm': 14.874964714050293, 'learning_rate': 8.756718753034381e-07, 'epoch': 0.06346623270951994, 'num_input_tokens_seen': 2168455168, 'completed': '28.04% (1_034 / 3_687)', 'remaining time': '23:30:43', 'throughput': '8392.21', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 17:32:49,717 >> {'loss': 0.4122, 'grad_norm': 15.657155990600586, 'learning_rate': 8.75393159478738e-07, 'epoch': 0.06373745592622729, 'num_input_tokens_seen': 2170552320, 'completed': '28.07% (1_035 / 3_687)', 'remaining time': '23:30:05', 'throughput': '8355.73', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 17:33:20,213 >> {'loss': 0.5967, 'grad_norm': 20.458627700805664, 'learning_rate': 8.751141817900052e-07, 'epoch': 0.06400867914293464, 'num_input_tokens_seen': 2172649472, 'completed': '28.10% (1_036 / 3_687)', 'remaining time': '23:29:17', 'throughput': '8596.13', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 17:33:53,339 >> {'loss': 0.2614, 'grad_norm': 11.38912296295166, 'learning_rate': 8.748349424617504e-07, 'epoch': 0.06427990235964198, 'num_input_tokens_seen': 2174746624, 'completed': '28.13% (1_037 / 3_687)', 'remaining time': '23:28:59', 'throughput': '7913.33', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 17:34:24,599 >> {'loss': 0.6497, 'grad_norm': 20.700756072998047, 'learning_rate': 8.745554417186946e-07, 'epoch': 0.06455112557634933, 'num_input_tokens_seen': 2176843776, 'completed': '28.15% (1_038 / 3_687)', 'remaining time': '23:28:20', 'throughput': '8386.04', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 17:34:53,913 >> {'loss': 0.7146, 'grad_norm': 20.341596603393555, 'learning_rate': 8.742756797857698e-07, 'epoch': 0.06482234879305669, 'num_input_tokens_seen': 2178940928, 'completed': '28.18% (1_039 / 3_687)', 'remaining time': '23:27:19', 'throughput': '8942.49', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 17:35:26,048 >> {'loss': 0.4446, 'grad_norm': 17.363826751708984, 'learning_rate': 8.739956568881174e-07, 'epoch': 0.06509357200976404, 'num_input_tokens_seen': 2181038080, 'completed': '28.21% (1_040 / 3_687)', 'remaining time': '23:26:50', 'throughput': '8157.63', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 17:35:58,225 >> {'loss': 0.4774, 'grad_norm': 21.04477882385254, 'learning_rate': 8.737153732510894e-07, 'epoch': 0.06536479522647139, 'num_input_tokens_seen': 2183135232, 'completed': '28.23% (1_041 / 3_687)', 'remaining time': '23:26:22', 'throughput': '8147.05', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 17:36:27,572 >> {'loss': 0.4042, 'grad_norm': 13.529879570007324, 'learning_rate': 8.734348291002472e-07, 'epoch': 0.06563601844317873, 'num_input_tokens_seen': 2185232384, 'completed': '28.26% (1_042 / 3_687)', 'remaining time': '23:25:22', 'throughput': '8932.46', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 17:36:59,615 >> {'loss': 0.6437, 'grad_norm': 17.152034759521484, 'learning_rate': 8.731540246613621e-07, 'epoch': 0.06590724165988608, 'num_input_tokens_seen': 2187329536, 'completed': '28.29% (1_043 / 3_687)', 'remaining time': '23:24:52', 'throughput': '8181.00', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 17:37:30,757 >> {'loss': 0.2645, 'grad_norm': 11.518980979919434, 'learning_rate': 8.728729601604149e-07, 'epoch': 0.06617846487659344, 'num_input_tokens_seen': 2189426688, 'completed': '28.32% (1_044 / 3_687)', 'remaining time': '23:24:12', 'throughput': '8417.72', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 17:38:00,930 >> {'loss': 0.5292, 'grad_norm': 19.25179100036621, 'learning_rate': 8.725916358235956e-07, 'epoch': 0.06644968809330079, 'num_input_tokens_seen': 2191523840, 'completed': '28.34% (1_045 / 3_687)', 'remaining time': '23:23:22', 'throughput': '8688.03', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 17:38:27,912 >> {'loss': 0.6961, 'grad_norm': 19.196977615356445, 'learning_rate': 8.723100518773034e-07, 'epoch': 0.06672091131000814, 'num_input_tokens_seen': 2193620992, 'completed': '28.37% (1_046 / 3_687)', 'remaining time': '23:21:57', 'throughput': '9715.47', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 17:38:58,751 >> {'loss': 0.3306, 'grad_norm': 12.074904441833496, 'learning_rate': 8.720282085481463e-07, 'epoch': 0.06699213452671549, 'num_input_tokens_seen': 2195718144, 'completed': '28.40% (1_047 / 3_687)', 'remaining time': '23:21:15', 'throughput': '8500.54', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 17:39:30,226 >> {'loss': 0.4436, 'grad_norm': 15.995697975158691, 'learning_rate': 8.717461060629408e-07, 'epoch': 0.06726335774342283, 'num_input_tokens_seen': 2197815296, 'completed': '28.42% (1_048 / 3_687)', 'remaining time': '23:20:39', 'throughput': '8328.59', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 17:40:00,395 >> {'loss': 0.5042, 'grad_norm': 16.870420455932617, 'learning_rate': 8.714637446487127e-07, 'epoch': 0.06753458096013018, 'num_input_tokens_seen': 2199912448, 'completed': '28.45% (1_049 / 3_687)', 'remaining time': '23:19:49', 'throughput': '8689.21', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 17:40:29,814 >> {'loss': 0.4686, 'grad_norm': 16.621347427368164, 'learning_rate': 8.711811245326955e-07, 'epoch': 0.06780580417683754, 'num_input_tokens_seen': 2202009600, 'completed': '28.48% (1_050 / 3_687)', 'remaining time': '23:18:52', 'throughput': '8910.67', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 17:41:02,490 >> {'loss': 0.5436, 'grad_norm': 20.809621810913086, 'learning_rate': 8.70898245942331e-07, 'epoch': 0.06807702739354489, 'num_input_tokens_seen': 2204106752, 'completed': '28.51% (1_051 / 3_687)', 'remaining time': '23:18:29', 'throughput': '8022.41', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 17:41:31,667 >> {'loss': 0.5587, 'grad_norm': 18.218713760375977, 'learning_rate': 8.706151091052693e-07, 'epoch': 0.06834825061025224, 'num_input_tokens_seen': 2206203904, 'completed': '28.53% (1_052 / 3_687)', 'remaining time': '23:17:29', 'throughput': '8984.88', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 17:42:03,986 >> {'loss': 0.5315, 'grad_norm': 15.873334884643555, 'learning_rate': 8.703317142493681e-07, 'epoch': 0.06861947382695958, 'num_input_tokens_seen': 2208301056, 'completed': '28.56% (1_053 / 3_687)', 'remaining time': '23:17:03', 'throughput': '8111.05', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 17:42:32,071 >> {'loss': 0.6862, 'grad_norm': 22.45003890991211, 'learning_rate': 8.700480616026928e-07, 'epoch': 0.06889069704366693, 'num_input_tokens_seen': 2210398208, 'completed': '28.59% (1_054 / 3_687)', 'remaining time': '23:15:52', 'throughput': '9333.96', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 17:43:03,264 >> {'loss': 0.5649, 'grad_norm': 23.83209991455078, 'learning_rate': 8.697641513935164e-07, 'epoch': 0.0691619202603743, 'num_input_tokens_seen': 2212495360, 'completed': '28.61% (1_055 / 3_687)', 'remaining time': '23:15:14', 'throughput': '8403.91', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 17:43:32,681 >> {'loss': 0.5258, 'grad_norm': 16.99089241027832, 'learning_rate': 8.694799838503186e-07, 'epoch': 0.06943314347708164, 'num_input_tokens_seen': 2214592512, 'completed': '28.64% (1_056 / 3_687)', 'remaining time': '23:14:18', 'throughput': '8911.38', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 17:44:02,885 >> {'loss': 0.3312, 'grad_norm': 12.274675369262695, 'learning_rate': 8.691955592017872e-07, 'epoch': 0.06970436669378899, 'num_input_tokens_seen': 2216689664, 'completed': '28.67% (1_057 / 3_687)', 'remaining time': '23:13:29', 'throughput': '8679.04', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 17:44:28,508 >> {'loss': 0.6978, 'grad_norm': 20.667945861816406, 'learning_rate': 8.689108776768159e-07, 'epoch': 0.06997558991049634, 'num_input_tokens_seen': 2218786816, 'completed': '28.70% (1_058 / 3_687)', 'remaining time': '23:11:55', 'throughput': '10230.70', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 17:44:56,984 >> {'loss': 0.6072, 'grad_norm': 18.657791137695312, 'learning_rate': 8.686259395045056e-07, 'epoch': 0.07024681312720368, 'num_input_tokens_seen': 2220883968, 'completed': '28.72% (1_059 / 3_687)', 'remaining time': '23:10:50', 'throughput': '9205.76', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 17:45:29,116 >> {'loss': 0.4635, 'grad_norm': 15.566442489624023, 'learning_rate': 8.68340744914164e-07, 'epoch': 0.07051803634391104, 'num_input_tokens_seen': 2222981120, 'completed': '28.75% (1_060 / 3_687)', 'remaining time': '23:10:22', 'throughput': '8158.48', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 17:45:59,163 >> {'loss': 0.2827, 'grad_norm': 14.146040916442871, 'learning_rate': 8.680552941353045e-07, 'epoch': 0.07078925956061839, 'num_input_tokens_seen': 2225078272, 'completed': '28.78% (1_061 / 3_687)', 'remaining time': '23:09:33', 'throughput': '8724.28', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 17:46:29,381 >> {'loss': 0.6515, 'grad_norm': 22.9758358001709, 'learning_rate': 8.677695873976473e-07, 'epoch': 0.07106048277732574, 'num_input_tokens_seen': 2227175424, 'completed': '28.80% (1_062 / 3_687)', 'remaining time': '23:08:46', 'throughput': '8675.26', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 17:47:00,027 >> {'loss': 0.4001, 'grad_norm': 14.749287605285645, 'learning_rate': 8.674836249311182e-07, 'epoch': 0.07133170599403309, 'num_input_tokens_seen': 2229272576, 'completed': '28.83% (1_063 / 3_687)', 'remaining time': '23:08:03', 'throughput': '8554.00', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 17:47:27,607 >> {'loss': 0.6569, 'grad_norm': 18.237518310546875, 'learning_rate': 8.671974069658488e-07, 'epoch': 0.07160292921074043, 'num_input_tokens_seen': 2231369728, 'completed': '28.86% (1_064 / 3_687)', 'remaining time': '23:06:50', 'throughput': '9504.80', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 17:47:56,271 >> {'loss': 0.5926, 'grad_norm': 21.23828125, 'learning_rate': 8.669109337321767e-07, 'epoch': 0.0718741524274478, 'num_input_tokens_seen': 2233466880, 'completed': '28.89% (1_065 / 3_687)', 'remaining time': '23:05:48', 'throughput': '9145.24', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 17:48:25,236 >> {'loss': 0.4136, 'grad_norm': 13.614550590515137, 'learning_rate': 8.666242054606444e-07, 'epoch': 0.07214537564415514, 'num_input_tokens_seen': 2235564032, 'completed': '28.91% (1_066 / 3_687)', 'remaining time': '23:04:49', 'throughput': '9050.56', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 17:48:57,758 >> {'loss': 0.5478, 'grad_norm': 19.72222900390625, 'learning_rate': 8.66337222382e-07, 'epoch': 0.07241659886086249, 'num_input_tokens_seen': 2237661184, 'completed': '28.94% (1_067 / 3_687)', 'remaining time': '23:04:25', 'throughput': '8060.56', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 17:49:27,012 >> {'loss': 0.7417, 'grad_norm': 24.28289222717285, 'learning_rate': 8.660499847271965e-07, 'epoch': 0.07268782207756984, 'num_input_tokens_seen': 2239758336, 'completed': '28.97% (1_068 / 3_687)', 'remaining time': '23:03:30', 'throughput': '8960.92', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 17:49:58,909 >> {'loss': 0.3927, 'grad_norm': 15.440895080566406, 'learning_rate': 8.657624927273919e-07, 'epoch': 0.07295904529427719, 'num_input_tokens_seen': 2241855488, 'completed': '28.99% (1_069 / 3_687)', 'remaining time': '23:03:00', 'throughput': '8218.48', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 17:50:28,609 >> {'loss': 0.4269, 'grad_norm': 14.023824691772461, 'learning_rate': 8.654747466139488e-07, 'epoch': 0.07323026851098453, 'num_input_tokens_seen': 2243952640, 'completed': '29.02% (1_070 / 3_687)', 'remaining time': '23:02:09', 'throughput': '8826.23', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 17:50:57,534 >> {'loss': 0.5167, 'grad_norm': 25.52799415588379, 'learning_rate': 8.651867466184344e-07, 'epoch': 0.0735014917276919, 'num_input_tokens_seen': 2246049792, 'completed': '29.05% (1_071 / 3_687)', 'remaining time': '23:01:11', 'throughput': '9063.02', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 17:51:28,804 >> {'loss': 0.4347, 'grad_norm': 15.417862892150879, 'learning_rate': 8.6489849297262e-07, 'epoch': 0.07377271494439924, 'num_input_tokens_seen': 2248146944, 'completed': '29.08% (1_072 / 3_687)', 'remaining time': '23:00:35', 'throughput': '8383.22', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 17:51:59,433 >> {'loss': 0.4029, 'grad_norm': 17.311777114868164, 'learning_rate': 8.646099859084812e-07, 'epoch': 0.07404393816110659, 'num_input_tokens_seen': 2250244096, 'completed': '29.10% (1_073 / 3_687)', 'remaining time': '22:59:53', 'throughput': '8558.52', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 17:52:28,669 >> {'loss': 0.4559, 'grad_norm': 15.483137130737305, 'learning_rate': 8.643212256581978e-07, 'epoch': 0.07431516137781394, 'num_input_tokens_seen': 2252341248, 'completed': '29.13% (1_074 / 3_687)', 'remaining time': '22:58:58', 'throughput': '8966.62', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 17:52:58,955 >> {'loss': 0.4146, 'grad_norm': 18.89014434814453, 'learning_rate': 8.640322124541525e-07, 'epoch': 0.07458638459452128, 'num_input_tokens_seen': 2254438400, 'completed': '29.16% (1_075 / 3_687)', 'remaining time': '22:58:14', 'throughput': '8655.55', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 17:53:28,941 >> {'loss': 0.7084, 'grad_norm': 23.15615463256836, 'learning_rate': 8.637429465289324e-07, 'epoch': 0.07485760781122865, 'num_input_tokens_seen': 2256535552, 'completed': '29.18% (1_076 / 3_687)', 'remaining time': '22:57:26', 'throughput': '8742.28', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 17:53:57,328 >> {'loss': 0.9022, 'grad_norm': 26.009380340576172, 'learning_rate': 8.63453428115328e-07, 'epoch': 0.075128831027936, 'num_input_tokens_seen': 2258632704, 'completed': '29.21% (1_077 / 3_687)', 'remaining time': '22:56:24', 'throughput': '9234.74', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 17:54:24,859 >> {'loss': 0.9285, 'grad_norm': 28.278987884521484, 'learning_rate': 8.631636574463321e-07, 'epoch': 0.07540005424464334, 'num_input_tokens_seen': 2260729856, 'completed': '29.24% (1_078 / 3_687)', 'remaining time': '22:55:14', 'throughput': '9521.85', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 17:54:54,217 >> {'loss': 0.5127, 'grad_norm': 15.612220764160156, 'learning_rate': 8.628736347551417e-07, 'epoch': 0.07567127746135069, 'num_input_tokens_seen': 2262827008, 'completed': '29.26% (1_079 / 3_687)', 'remaining time': '22:54:21', 'throughput': '8928.97', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 17:55:27,538 >> {'loss': 0.4181, 'grad_norm': 15.422523498535156, 'learning_rate': 8.625833602751559e-07, 'epoch': 0.07594250067805804, 'num_input_tokens_seen': 2264924160, 'completed': '29.29% (1_080 / 3_687)', 'remaining time': '22:54:05', 'throughput': '7867.35', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 17:55:56,476 >> {'loss': 0.9629, 'grad_norm': 26.36939239501953, 'learning_rate': 8.622928342399762e-07, 'epoch': 0.0762137238947654, 'num_input_tokens_seen': 2267021312, 'completed': '29.32% (1_081 / 3_687)', 'remaining time': '22:53:08', 'throughput': '9058.66', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 17:56:26,134 >> {'loss': 0.6093, 'grad_norm': 17.090736389160156, 'learning_rate': 8.620020568834072e-07, 'epoch': 0.07648494711147275, 'num_input_tokens_seen': 2269118464, 'completed': '29.35% (1_082 / 3_687)', 'remaining time': '22:52:19', 'throughput': '8838.98', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 17:56:54,704 >> {'loss': 0.5369, 'grad_norm': 22.48678207397461, 'learning_rate': 8.617110284394553e-07, 'epoch': 0.07675617032818009, 'num_input_tokens_seen': 2271215616, 'completed': '29.37% (1_083 / 3_687)', 'remaining time': '22:51:19', 'throughput': '9175.50', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 17:57:23,909 >> {'loss': 0.3614, 'grad_norm': 16.42003059387207, 'learning_rate': 8.614197491423293e-07, 'epoch': 0.07702739354488744, 'num_input_tokens_seen': 2273312768, 'completed': '29.40% (1_084 / 3_687)', 'remaining time': '22:50:26', 'throughput': '8976.12', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 17:57:52,585 >> {'loss': 0.7799, 'grad_norm': 45.502445220947266, 'learning_rate': 8.611282192264396e-07, 'epoch': 0.07729861676159479, 'num_input_tokens_seen': 2275409920, 'completed': '29.43% (1_085 / 3_687)', 'remaining time': '22:49:27', 'throughput': '9141.60', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 17:58:22,285 >> {'loss': 0.3233, 'grad_norm': 11.837736129760742, 'learning_rate': 8.608364389263984e-07, 'epoch': 0.07756983997830215, 'num_input_tokens_seen': 2277507072, 'completed': '29.45% (1_086 / 3_687)', 'remaining time': '22:48:39', 'throughput': '8826.39', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 17:58:54,699 >> {'loss': 0.5218, 'grad_norm': 18.987957000732422, 'learning_rate': 8.605444084770192e-07, 'epoch': 0.0778410631950095, 'num_input_tokens_seen': 2279604224, 'completed': '29.48% (1_087 / 3_687)', 'remaining time': '22:48:15', 'throughput': '8087.20', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 17:59:25,308 >> {'loss': 0.4539, 'grad_norm': 17.756649017333984, 'learning_rate': 8.602521281133173e-07, 'epoch': 0.07811228641171684, 'num_input_tokens_seen': 2281701376, 'completed': '29.51% (1_088 / 3_687)', 'remaining time': '22:47:35', 'throughput': '8564.31', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 17:59:55,680 >> {'loss': 0.2876, 'grad_norm': 21.006437301635742, 'learning_rate': 8.599595980705085e-07, 'epoch': 0.07838350962842419, 'num_input_tokens_seen': 2283798528, 'completed': '29.54% (1_089 / 3_687)', 'remaining time': '22:46:52', 'throughput': '8631.11', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 18:00:28,369 >> {'loss': 0.5252, 'grad_norm': 17.649852752685547, 'learning_rate': 8.596668185840102e-07, 'epoch': 0.07865473284513154, 'num_input_tokens_seen': 2285895680, 'completed': '29.56% (1_090 / 3_687)', 'remaining time': '22:46:31', 'throughput': '8019.32', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 18:00:56,463 >> {'loss': 0.5004, 'grad_norm': 15.316368103027344, 'learning_rate': 8.593737898894398e-07, 'epoch': 0.07892595606183889, 'num_input_tokens_seen': 2287992832, 'completed': '29.59% (1_091 / 3_687)', 'remaining time': '22:45:28', 'throughput': '9331.08', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 18:01:23,674 >> {'loss': 0.5126, 'grad_norm': 16.443462371826172, 'learning_rate': 8.59080512222616e-07, 'epoch': 0.07919717927854625, 'num_input_tokens_seen': 2290089984, 'completed': '29.62% (1_092 / 3_687)', 'remaining time': '22:44:18', 'throughput': '9633.54', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 18:01:57,430 >> {'loss': 0.6624, 'grad_norm': 25.65877342224121, 'learning_rate': 8.587869858195574e-07, 'epoch': 0.0794684024952536, 'num_input_tokens_seen': 2292187136, 'completed': '29.64% (1_093 / 3_687)', 'remaining time': '22:44:06', 'throughput': '7765.85', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 18:02:26,558 >> {'loss': 0.5005, 'grad_norm': 15.421908378601074, 'learning_rate': 8.584932109164826e-07, 'epoch': 0.07973962571196094, 'num_input_tokens_seen': 2294284288, 'completed': '29.67% (1_094 / 3_687)', 'remaining time': '22:43:13', 'throughput': '8999.79', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 18:03:00,897 >> {'loss': 0.3672, 'grad_norm': 12.648368835449219, 'learning_rate': 8.581991877498109e-07, 'epoch': 0.08001084892866829, 'num_input_tokens_seen': 2296381440, 'completed': '29.70% (1_095 / 3_687)', 'remaining time': '22:43:06', 'throughput': '7634.14', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 18:03:29,569 >> {'loss': 0.6201, 'grad_norm': 20.8604679107666, 'learning_rate': 8.579049165561607e-07, 'epoch': 0.08028207214537564, 'num_input_tokens_seen': 2298478592, 'completed': '29.73% (1_096 / 3_687)', 'remaining time': '22:42:09', 'throughput': '9142.75', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 18:03:58,166 >> {'loss': 0.6181, 'grad_norm': 18.767131805419922, 'learning_rate': 8.576103975723502e-07, 'epoch': 0.080553295362083, 'num_input_tokens_seen': 2300575744, 'completed': '29.75% (1_097 / 3_687)', 'remaining time': '22:41:12', 'throughput': '9166.72', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 18:04:27,054 >> {'loss': 0.5909, 'grad_norm': 18.214879989624023, 'learning_rate': 8.573156310353974e-07, 'epoch': 0.08082451857879035, 'num_input_tokens_seen': 2302672896, 'completed': '29.78% (1_098 / 3_687)', 'remaining time': '22:40:17', 'throughput': '9074.70', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 18:04:53,968 >> {'loss': 0.5828, 'grad_norm': 15.940201759338379, 'learning_rate': 8.570206171825188e-07, 'epoch': 0.0810957417954977, 'num_input_tokens_seen': 2304770048, 'completed': '29.81% (1_099 / 3_687)', 'remaining time': '22:39:06', 'throughput': '9739.78', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 18:05:24,136 >> {'loss': 0.4176, 'grad_norm': 13.9312744140625, 'learning_rate': 8.567253562511306e-07, 'epoch': 0.08136696501220504, 'num_input_tokens_seen': 2306867200, 'completed': '29.83% (1_100 / 3_687)', 'remaining time': '22:38:23', 'throughput': '8689.72', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 18:05:53,322 >> {'loss': 0.4367, 'grad_norm': 16.388742446899414, 'learning_rate': 8.564298484788472e-07, 'epoch': 0.08163818822891239, 'num_input_tokens_seen': 2308964352, 'completed': '29.86% (1_101 / 3_687)', 'remaining time': '22:37:32', 'throughput': '8981.71', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 18:06:24,120 >> {'loss': 0.2912, 'grad_norm': 11.300383567810059, 'learning_rate': 8.561340941034825e-07, 'epoch': 0.08190941144561975, 'num_input_tokens_seen': 2311061504, 'completed': '29.89% (1_102 / 3_687)', 'remaining time': '22:36:54', 'throughput': '8511.79', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 18:06:54,937 >> {'loss': 0.6453, 'grad_norm': 19.192779541015625, 'learning_rate': 8.55838093363048e-07, 'epoch': 0.0821806346623271, 'num_input_tokens_seen': 2313158656, 'completed': '29.92% (1_103 / 3_687)', 'remaining time': '22:36:17', 'throughput': '8506.35', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 18:07:26,525 >> {'loss': 0.6856, 'grad_norm': 20.092891693115234, 'learning_rate': 8.555418464957542e-07, 'epoch': 0.08245185787903445, 'num_input_tokens_seen': 2315255808, 'completed': '29.94% (1_104 / 3_687)', 'remaining time': '22:35:46', 'throughput': '8298.90', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 18:07:55,385 >> {'loss': 0.4858, 'grad_norm': 16.484115600585938, 'learning_rate': 8.552453537400089e-07, 'epoch': 0.0827230810957418, 'num_input_tokens_seen': 2317352960, 'completed': '29.97% (1_105 / 3_687)', 'remaining time': '22:34:52', 'throughput': '9083.45', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 18:08:27,003 >> {'loss': 0.7113, 'grad_norm': 27.20563316345215, 'learning_rate': 8.549486153344183e-07, 'epoch': 0.08299430431244914, 'num_input_tokens_seen': 2319450112, 'completed': '30.00% (1_106 / 3_687)', 'remaining time': '22:34:22', 'throughput': '8290.74', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 18:08:54,435 >> {'loss': 0.6143, 'grad_norm': 17.81060028076172, 'learning_rate': 8.546516315177863e-07, 'epoch': 0.0832655275291565, 'num_input_tokens_seen': 2321547264, 'completed': '30.02% (1_107 / 3_687)', 'remaining time': '22:33:16', 'throughput': '9556.40', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 18:09:25,059 >> {'loss': 0.5014, 'grad_norm': 17.569068908691406, 'learning_rate': 8.543544025291143e-07, 'epoch': 0.08353675074586385, 'num_input_tokens_seen': 2323644416, 'completed': '30.05% (1_108 / 3_687)', 'remaining time': '22:32:38', 'throughput': '8560.07', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 18:09:55,155 >> {'loss': 0.4685, 'grad_norm': 15.904462814331055, 'learning_rate': 8.540569286076004e-07, 'epoch': 0.0838079739625712, 'num_input_tokens_seen': 2325741568, 'completed': '30.08% (1_109 / 3_687)', 'remaining time': '22:31:55', 'throughput': '8710.23', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 18:10:26,219 >> {'loss': 0.6062, 'grad_norm': 19.58460807800293, 'learning_rate': 8.537592099926407e-07, 'epoch': 0.08407919717927854, 'num_input_tokens_seen': 2327838720, 'completed': '30.11% (1_110 / 3_687)', 'remaining time': '22:31:20', 'throughput': '8438.71', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 18:10:58,092 >> {'loss': 0.3705, 'grad_norm': 13.513802528381348, 'learning_rate': 8.534612469238278e-07, 'epoch': 0.08435042039598589, 'num_input_tokens_seen': 2329935872, 'completed': '30.13% (1_111 / 3_687)', 'remaining time': '22:30:52', 'throughput': '8224.77', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 18:11:28,820 >> {'loss': 0.677, 'grad_norm': 20.021507263183594, 'learning_rate': 8.531630396409507e-07, 'epoch': 0.08462164361269324, 'num_input_tokens_seen': 2332033024, 'completed': '30.16% (1_112 / 3_687)', 'remaining time': '22:30:14', 'throughput': '8531.19', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 18:12:00,262 >> {'loss': 0.5015, 'grad_norm': 14.840436935424805, 'learning_rate': 8.528645883839956e-07, 'epoch': 0.0848928668294006, 'num_input_tokens_seen': 2334130176, 'completed': '30.19% (1_113 / 3_687)', 'remaining time': '22:29:43', 'throughput': '8337.19', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 18:12:31,096 >> {'loss': 0.5022, 'grad_norm': 15.523248672485352, 'learning_rate': 8.525658933931448e-07, 'epoch': 0.08516409004610795, 'num_input_tokens_seen': 2336227328, 'completed': '30.21% (1_114 / 3_687)', 'remaining time': '22:29:06', 'throughput': '8501.75', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 18:12:58,027 >> {'loss': 0.5963, 'grad_norm': 17.14645004272461, 'learning_rate': 8.522669549087762e-07, 'epoch': 0.0854353132628153, 'num_input_tokens_seen': 2338324480, 'completed': '30.24% (1_115 / 3_687)', 'remaining time': '22:27:58', 'throughput': '9734.04', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 18:13:28,648 >> {'loss': 0.7184, 'grad_norm': 20.818403244018555, 'learning_rate': 8.519677731714645e-07, 'epoch': 0.08570653647952264, 'num_input_tokens_seen': 2340421632, 'completed': '30.27% (1_116 / 3_687)', 'remaining time': '22:27:20', 'throughput': '8561.04', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 18:13:58,333 >> {'loss': 0.4394, 'grad_norm': 13.36194896697998, 'learning_rate': 8.516683484219797e-07, 'epoch': 0.08597775969622999, 'num_input_tokens_seen': 2342518784, 'completed': '30.30% (1_117 / 3_687)', 'remaining time': '22:26:34', 'throughput': '8830.88', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 18:14:29,050 >> {'loss': 0.5567, 'grad_norm': 18.085691452026367, 'learning_rate': 8.513686809012875e-07, 'epoch': 0.08624898291293735, 'num_input_tokens_seen': 2344615936, 'completed': '30.32% (1_118 / 3_687)', 'remaining time': '22:25:57', 'throughput': '8533.92', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 18:14:58,383 >> {'loss': 0.5698, 'grad_norm': 18.329633712768555, 'learning_rate': 8.510687708505489e-07, 'epoch': 0.0865202061296447, 'num_input_tokens_seen': 2346713088, 'completed': '30.35% (1_119 / 3_687)', 'remaining time': '22:25:08', 'throughput': '8936.88', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 18:15:28,760 >> {'loss': 0.5401, 'grad_norm': 23.978832244873047, 'learning_rate': 8.507686185111199e-07, 'epoch': 0.08679142934635205, 'num_input_tokens_seen': 2348810240, 'completed': '30.38% (1_120 / 3_687)', 'remaining time': '22:24:28', 'throughput': '8629.76', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 18:15:57,861 >> {'loss': 0.3782, 'grad_norm': 19.02138328552246, 'learning_rate': 8.504682241245516e-07, 'epoch': 0.0870626525630594, 'num_input_tokens_seen': 2350907392, 'completed': '30.40% (1_121 / 3_687)', 'remaining time': '22:23:38', 'throughput': '9007.98', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 18:16:27,778 >> {'loss': 0.2031, 'grad_norm': 10.588142395019531, 'learning_rate': 8.501675879325906e-07, 'epoch': 0.08733387577976674, 'num_input_tokens_seen': 2353004544, 'completed': '30.43% (1_122 / 3_687)', 'remaining time': '22:22:55', 'throughput': '8762.53', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 18:16:59,488 >> {'loss': 0.4395, 'grad_norm': 18.56726837158203, 'learning_rate': 8.498667101771769e-07, 'epoch': 0.0876050989964741, 'num_input_tokens_seen': 2355101696, 'completed': '30.46% (1_123 / 3_687)', 'remaining time': '22:22:26', 'throughput': '8266.81', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 18:17:32,899 >> {'loss': 0.6843, 'grad_norm': 21.876890182495117, 'learning_rate': 8.495655911004456e-07, 'epoch': 0.08787632221318145, 'num_input_tokens_seen': 2357198848, 'completed': '30.49% (1_124 / 3_687)', 'remaining time': '22:22:10', 'throughput': '7846.09', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 18:18:03,622 >> {'loss': 0.508, 'grad_norm': 17.283998489379883, 'learning_rate': 8.492642309447257e-07, 'epoch': 0.0881475454298888, 'num_input_tokens_seen': 2359296000, 'completed': '30.51% (1_125 / 3_687)', 'remaining time': '22:21:33', 'throughput': '8532.36', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 18:18:33,629 >> {'loss': 0.3963, 'grad_norm': 12.905929565429688, 'learning_rate': 8.489626299525409e-07, 'epoch': 0.08841876864659615, 'num_input_tokens_seen': 2361393152, 'completed': '30.54% (1_126 / 3_687)', 'remaining time': '22:20:51', 'throughput': '8736.28', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 18:19:03,545 >> {'loss': 0.4069, 'grad_norm': 14.936822891235352, 'learning_rate': 8.486607883666077e-07, 'epoch': 0.0886899918633035, 'num_input_tokens_seen': 2363490304, 'completed': '30.57% (1_127 / 3_687)', 'remaining time': '22:20:08', 'throughput': '8762.50', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 18:19:34,742 >> {'loss': 0.5104, 'grad_norm': 16.91001319885254, 'learning_rate': 8.483587064298372e-07, 'epoch': 0.08896121508001086, 'num_input_tokens_seen': 2365587456, 'completed': '30.59% (1_128 / 3_687)', 'remaining time': '22:19:35', 'throughput': '8402.94', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 18:20:07,848 >> {'loss': 0.3908, 'grad_norm': 15.12585735321045, 'learning_rate': 8.480563843853328e-07, 'epoch': 0.0892324382967182, 'num_input_tokens_seen': 2367684608, 'completed': '30.62% (1_129 / 3_687)', 'remaining time': '22:19:17', 'throughput': '7918.40', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 18:20:38,884 >> {'loss': 0.6482, 'grad_norm': 25.309404373168945, 'learning_rate': 8.477538224763923e-07, 'epoch': 0.08950366151342555, 'num_input_tokens_seen': 2369781760, 'completed': '30.65% (1_130 / 3_687)', 'remaining time': '22:18:42', 'throughput': '8446.36', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 18:21:09,859 >> {'loss': 0.4492, 'grad_norm': 26.27945899963379, 'learning_rate': 8.474510209465058e-07, 'epoch': 0.0897748847301329, 'num_input_tokens_seen': 2371878912, 'completed': '30.68% (1_131 / 3_687)', 'remaining time': '22:18:07', 'throughput': '8463.04', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 18:21:40,840 >> {'loss': 0.545, 'grad_norm': 19.251035690307617, 'learning_rate': 8.471479800393565e-07, 'epoch': 0.09004610794684025, 'num_input_tokens_seen': 2373976064, 'completed': '30.70% (1_132 / 3_687)', 'remaining time': '22:17:33', 'throughput': '8461.63', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 18:22:12,496 >> {'loss': 0.5393, 'grad_norm': 19.276018142700195, 'learning_rate': 8.468446999988202e-07, 'epoch': 0.09031733116354759, 'num_input_tokens_seen': 2376073216, 'completed': '30.73% (1_133 / 3_687)', 'remaining time': '22:17:03', 'throughput': '8281.13', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 18:22:38,222 >> {'loss': 0.9826, 'grad_norm': 25.423450469970703, 'learning_rate': 8.465411810689653e-07, 'epoch': 0.09058855438025495, 'num_input_tokens_seen': 2378170368, 'completed': '30.76% (1_134 / 3_687)', 'remaining time': '22:15:48', 'throughput': '10189.71', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 18:23:09,214 >> {'loss': 0.487, 'grad_norm': 17.12239646911621, 'learning_rate': 8.462374234940517e-07, 'epoch': 0.0908597775969623, 'num_input_tokens_seen': 2380267520, 'completed': '30.78% (1_135 / 3_687)', 'remaining time': '22:15:14', 'throughput': '8458.33', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 18:23:38,886 >> {'loss': 0.3742, 'grad_norm': 14.614494323730469, 'learning_rate': 8.459334275185325e-07, 'epoch': 0.09113100081366965, 'num_input_tokens_seen': 2382364672, 'completed': '30.81% (1_136 / 3_687)', 'remaining time': '22:14:29', 'throughput': '8834.77', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 18:24:07,126 >> {'loss': 1.0046, 'grad_norm': 27.48741340637207, 'learning_rate': 8.456291933870521e-07, 'epoch': 0.091402224030377, 'num_input_tokens_seen': 2384461824, 'completed': '30.84% (1_137 / 3_687)', 'remaining time': '22:13:34', 'throughput': '9282.57', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 18:24:38,578 >> {'loss': 0.2549, 'grad_norm': 12.554402351379395, 'learning_rate': 8.453247213444463e-07, 'epoch': 0.09167344724708434, 'num_input_tokens_seen': 2386558976, 'completed': '30.87% (1_138 / 3_687)', 'remaining time': '22:13:03', 'throughput': '8334.67', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 18:25:09,342 >> {'loss': 0.5053, 'grad_norm': 14.78433895111084, 'learning_rate': 8.450200116357428e-07, 'epoch': 0.0919446704637917, 'num_input_tokens_seen': 2388656128, 'completed': '30.89% (1_139 / 3_687)', 'remaining time': '22:12:27', 'throughput': '8521.32', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 18:25:41,290 >> {'loss': 0.3622, 'grad_norm': 12.076690673828125, 'learning_rate': 8.4471506450616e-07, 'epoch': 0.09221589368049905, 'num_input_tokens_seen': 2390753280, 'completed': '30.92% (1_140 / 3_687)', 'remaining time': '22:12:00', 'throughput': '8205.27', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 18:26:11,284 >> {'loss': 0.472, 'grad_norm': 18.700605392456055, 'learning_rate': 8.444098802011083e-07, 'epoch': 0.0924871168972064, 'num_input_tokens_seen': 2392850432, 'completed': '30.95% (1_141 / 3_687)', 'remaining time': '22:11:19', 'throughput': '8740.00', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 18:26:42,699 >> {'loss': 0.6367, 'grad_norm': 19.415464401245117, 'learning_rate': 8.441044589661881e-07, 'epoch': 0.09275834011391375, 'num_input_tokens_seen': 2394947584, 'completed': '30.97% (1_142 / 3_687)', 'remaining time': '22:10:47', 'throughput': '8344.39', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 18:27:13,078 >> {'loss': 0.51, 'grad_norm': 19.17083740234375, 'learning_rate': 8.437988010471907e-07, 'epoch': 0.0930295633306211, 'num_input_tokens_seen': 2397044736, 'completed': '31.00% (1_143 / 3_687)', 'remaining time': '22:10:09', 'throughput': '8629.32', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 18:27:43,326 >> {'loss': 0.533, 'grad_norm': 16.67664337158203, 'learning_rate': 8.434929066900982e-07, 'epoch': 0.09330078654732846, 'num_input_tokens_seen': 2399141888, 'completed': '31.03% (1_144 / 3_687)', 'remaining time': '22:09:29', 'throughput': '8666.30', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 18:28:10,973 >> {'loss': 0.8218, 'grad_norm': 20.507274627685547, 'learning_rate': 8.431867761410826e-07, 'epoch': 0.0935720097640358, 'num_input_tokens_seen': 2401239040, 'completed': '31.06% (1_145 / 3_687)', 'remaining time': '22:08:30', 'throughput': '9482.05', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 18:28:41,241 >> {'loss': 0.4415, 'grad_norm': 14.855219841003418, 'learning_rate': 8.42880409646506e-07, 'epoch': 0.09384323298074315, 'num_input_tokens_seen': 2403336192, 'completed': '31.08% (1_146 / 3_687)', 'remaining time': '22:07:51', 'throughput': '8660.66', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 18:29:12,361 >> {'loss': 0.485, 'grad_norm': 19.247512817382812, 'learning_rate': 8.42573807452921e-07, 'epoch': 0.0941144561974505, 'num_input_tokens_seen': 2405433344, 'completed': '31.11% (1_147 / 3_687)', 'remaining time': '22:07:18', 'throughput': '8423.92', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 18:29:41,653 >> {'loss': 0.5588, 'grad_norm': 23.100740432739258, 'learning_rate': 8.422669698070687e-07, 'epoch': 0.09438567941415785, 'num_input_tokens_seen': 2407530496, 'completed': '31.14% (1_148 / 3_687)', 'remaining time': '22:06:31', 'throughput': '8948.93', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 18:30:12,026 >> {'loss': 0.2426, 'grad_norm': 12.322291374206543, 'learning_rate': 8.419598969558808e-07, 'epoch': 0.09465690263086521, 'num_input_tokens_seen': 2409627648, 'completed': '31.16% (1_149 / 3_687)', 'remaining time': '22:05:53', 'throughput': '8631.02', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 18:30:44,036 >> {'loss': 0.3201, 'grad_norm': 11.358101844787598, 'learning_rate': 8.416525891464776e-07, 'epoch': 0.09492812584757256, 'num_input_tokens_seen': 2411724800, 'completed': '31.19% (1_150 / 3_687)', 'remaining time': '22:05:26', 'throughput': '8189.38', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 18:31:16,392 >> {'loss': 0.4272, 'grad_norm': 16.695714950561523, 'learning_rate': 8.413450466261691e-07, 'epoch': 0.0951993490642799, 'num_input_tokens_seen': 2413821952, 'completed': '31.22% (1_151 / 3_687)', 'remaining time': '22:05:02', 'throughput': '8101.81', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 18:31:46,647 >> {'loss': 0.3545, 'grad_norm': 14.923744201660156, 'learning_rate': 8.410372696424535e-07, 'epoch': 0.09547057228098725, 'num_input_tokens_seen': 2415919104, 'completed': '31.24% (1_152 / 3_687)', 'remaining time': '22:04:23', 'throughput': '8664.43', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 18:32:17,410 >> {'loss': 0.4019, 'grad_norm': 13.441437721252441, 'learning_rate': 8.40729258443018e-07, 'epoch': 0.0957417954976946, 'num_input_tokens_seen': 2418016256, 'completed': '31.27% (1_153 / 3_687)', 'remaining time': '22:03:48', 'throughput': '8521.57', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 18:32:46,540 >> {'loss': 0.3352, 'grad_norm': 13.833053588867188, 'learning_rate': 8.404210132757385e-07, 'epoch': 0.09601301871440195, 'num_input_tokens_seen': 2420113408, 'completed': '31.30% (1_154 / 3_687)', 'remaining time': '22:03:00', 'throughput': '8999.10', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 18:33:16,469 >> {'loss': 0.5448, 'grad_norm': 13.472502708435059, 'learning_rate': 8.401125343886787e-07, 'epoch': 0.09628424193110931, 'num_input_tokens_seen': 2422210560, 'completed': '31.33% (1_155 / 3_687)', 'remaining time': '22:02:19', 'throughput': '8758.76', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 18:33:46,833 >> {'loss': 0.3018, 'grad_norm': 10.587477684020996, 'learning_rate': 8.398038220300908e-07, 'epoch': 0.09655546514781665, 'num_input_tokens_seen': 2424307712, 'completed': '31.35% (1_156 / 3_687)', 'remaining time': '22:01:41', 'throughput': '8633.45', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 18:34:12,978 >> {'loss': 0.7406, 'grad_norm': 19.353105545043945, 'learning_rate': 8.39494876448415e-07, 'epoch': 0.096826688364524, 'num_input_tokens_seen': 2426404864, 'completed': '31.38% (1_157 / 3_687)', 'remaining time': '22:00:33', 'throughput': '10026.56', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 18:34:42,632 >> {'loss': 0.8482, 'grad_norm': 24.70726203918457, 'learning_rate': 8.391856978922785e-07, 'epoch': 0.09709791158123135, 'num_input_tokens_seen': 2428502016, 'completed': '31.41% (1_158 / 3_687)', 'remaining time': '21:59:50', 'throughput': '8840.15', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 18:35:13,337 >> {'loss': 0.232, 'grad_norm': 15.665159225463867, 'learning_rate': 8.38876286610497e-07, 'epoch': 0.0973691347979387, 'num_input_tokens_seen': 2430599168, 'completed': '31.43% (1_159 / 3_687)', 'remaining time': '21:59:14', 'throughput': '8537.42', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 18:35:43,400 >> {'loss': 0.6336, 'grad_norm': 19.37482452392578, 'learning_rate': 8.385666428520723e-07, 'epoch': 0.09764035801464606, 'num_input_tokens_seen': 2432696320, 'completed': '31.46% (1_160 / 3_687)', 'remaining time': '21:58:34', 'throughput': '8719.64', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 18:36:15,051 >> {'loss': 0.6754, 'grad_norm': 21.107303619384766, 'learning_rate': 8.382567668661943e-07, 'epoch': 0.0979115812313534, 'num_input_tokens_seen': 2434793472, 'completed': '31.49% (1_161 / 3_687)', 'remaining time': '21:58:05', 'throughput': '8282.56', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 18:36:45,077 >> {'loss': 0.613, 'grad_norm': 21.173263549804688, 'learning_rate': 8.379466589022393e-07, 'epoch': 0.09818280444806075, 'num_input_tokens_seen': 2436890624, 'completed': '31.52% (1_162 / 3_687)', 'remaining time': '21:57:25', 'throughput': '8730.34', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 18:37:15,381 >> {'loss': 0.5053, 'grad_norm': 23.236482620239258, 'learning_rate': 8.376363192097703e-07, 'epoch': 0.0984540276647681, 'num_input_tokens_seen': 2438987776, 'completed': '31.54% (1_163 / 3_687)', 'remaining time': '21:56:47', 'throughput': '8650.60', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 18:37:47,163 >> {'loss': 0.6246, 'grad_norm': 24.89634132385254, 'learning_rate': 8.37325748038537e-07, 'epoch': 0.09872525088147545, 'num_input_tokens_seen': 2441084928, 'completed': '31.57% (1_164 / 3_687)', 'remaining time': '21:56:19', 'throughput': '8248.04', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 18:38:16,304 >> {'loss': 0.4242, 'grad_norm': 17.755111694335938, 'learning_rate': 8.370149456384754e-07, 'epoch': 0.09899647409818281, 'num_input_tokens_seen': 2443182080, 'completed': '31.60% (1_165 / 3_687)', 'remaining time': '21:55:32', 'throughput': '8995.81', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 18:38:47,355 >> {'loss': 0.2753, 'grad_norm': 12.603349685668945, 'learning_rate': 8.36703912259707e-07, 'epoch': 0.09926769731489016, 'num_input_tokens_seen': 2445279232, 'completed': '31.62% (1_166 / 3_687)', 'remaining time': '21:54:59', 'throughput': '8442.46', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 18:39:16,796 >> {'loss': 0.7312, 'grad_norm': 24.28446388244629, 'learning_rate': 8.363926481525402e-07, 'epoch': 0.0995389205315975, 'num_input_tokens_seen': 2447376384, 'completed': '31.65% (1_167 / 3_687)', 'remaining time': '21:54:15', 'throughput': '8904.10', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 18:39:46,999 >> {'loss': 0.478, 'grad_norm': 16.006052017211914, 'learning_rate': 8.360811535674682e-07, 'epoch': 0.09981014374830485, 'num_input_tokens_seen': 2449473536, 'completed': '31.68% (1_168 / 3_687)', 'remaining time': '21:53:37', 'throughput': '8679.42', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 18:40:18,027 >> {'loss': 0.779, 'grad_norm': 25.193614959716797, 'learning_rate': 8.357694287551698e-07, 'epoch': 0.1000813669650122, 'num_input_tokens_seen': 2451570688, 'completed': '31.71% (1_169 / 3_687)', 'remaining time': '21:53:03', 'throughput': '8448.62', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 18:40:49,556 >> {'loss': 0.496, 'grad_norm': 14.870660781860352, 'learning_rate': 8.354574739665096e-07, 'epoch': 0.10035259018171956, 'num_input_tokens_seen': 2453667840, 'completed': '31.73% (1_170 / 3_687)', 'remaining time': '21:52:34', 'throughput': '8314.19', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 18:41:23,186 >> {'loss': 0.305, 'grad_norm': 17.222614288330078, 'learning_rate': 8.351452894525368e-07, 'epoch': 0.10062381339842691, 'num_input_tokens_seen': 2455764992, 'completed': '31.76% (1_171 / 3_687)', 'remaining time': '21:52:18', 'throughput': '7794.90', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 18:41:52,950 >> {'loss': 0.4074, 'grad_norm': 14.791529655456543, 'learning_rate': 8.348328754644855e-07, 'epoch': 0.10089503661513426, 'num_input_tokens_seen': 2457862144, 'completed': '31.79% (1_172 / 3_687)', 'remaining time': '21:51:37', 'throughput': '8807.42', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 18:42:22,545 >> {'loss': 0.6192, 'grad_norm': 20.521638870239258, 'learning_rate': 8.34520232253775e-07, 'epoch': 0.1011662598318416, 'num_input_tokens_seen': 2459959296, 'completed': '31.81% (1_173 / 3_687)', 'remaining time': '21:50:54', 'throughput': '8857.86', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 18:42:53,901 >> {'loss': 0.6189, 'grad_norm': 25.750980377197266, 'learning_rate': 8.342073600720082e-07, 'epoch': 0.10143748304854895, 'num_input_tokens_seen': 2462056448, 'completed': '31.84% (1_174 / 3_687)', 'remaining time': '21:50:23', 'throughput': '8360.24', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 18:43:30,368 >> {'loss': 0.477, 'grad_norm': 18.543386459350586, 'learning_rate': 8.33894259170973e-07, 'epoch': 0.10170870626525631, 'num_input_tokens_seen': 2464153600, 'completed': '31.87% (1_175 / 3_687)', 'remaining time': '21:50:27', 'throughput': '7188.46', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 18:44:02,140 >> {'loss': 0.3135, 'grad_norm': 13.476463317871094, 'learning_rate': 8.335809298026409e-07, 'epoch': 0.10197992948196366, 'num_input_tokens_seen': 2466250752, 'completed': '31.90% (1_176 / 3_687)', 'remaining time': '21:49:59', 'throughput': '8250.80', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 18:44:34,253 >> {'loss': 0.3671, 'grad_norm': 19.402223587036133, 'learning_rate': 8.332673722191677e-07, 'epoch': 0.10225115269867101, 'num_input_tokens_seen': 2468347904, 'completed': '31.92% (1_177 / 3_687)', 'remaining time': '21:49:33', 'throughput': '8163.18', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 18:45:04,811 >> {'loss': 0.4054, 'grad_norm': 21.820781707763672, 'learning_rate': 8.329535866728922e-07, 'epoch': 0.10252237591537836, 'num_input_tokens_seen': 2470445056, 'completed': '31.95% (1_178 / 3_687)', 'remaining time': '21:48:56', 'throughput': '8578.63', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 18:45:36,619 >> {'loss': 0.6286, 'grad_norm': 19.219013214111328, 'learning_rate': 8.326395734163375e-07, 'epoch': 0.1027935991320857, 'num_input_tokens_seen': 2472542208, 'completed': '31.98% (1_179 / 3_687)', 'remaining time': '21:48:28', 'throughput': '8241.44', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 18:46:12,339 >> {'loss': 0.4077, 'grad_norm': 22.508085250854492, 'learning_rate': 8.323253327022094e-07, 'epoch': 0.10306482234879305, 'num_input_tokens_seen': 2474639360, 'completed': '32.00% (1_180 / 3_687)', 'remaining time': '21:48:26', 'throughput': '7338.88', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 18:46:46,229 >> {'loss': 0.3092, 'grad_norm': 12.82617473602295, 'learning_rate': 8.320108647833967e-07, 'epoch': 0.10333604556550041, 'num_input_tokens_seen': 2476736512, 'completed': '32.03% (1_181 / 3_687)', 'remaining time': '21:48:12', 'throughput': '7735.22', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 18:47:16,554 >> {'loss': 0.2255, 'grad_norm': 12.152515411376953, 'learning_rate': 8.316961699129714e-07, 'epoch': 0.10360726878220776, 'num_input_tokens_seen': 2478833664, 'completed': '32.06% (1_182 / 3_687)', 'remaining time': '21:47:34', 'throughput': '8644.46', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 18:47:48,881 >> {'loss': 0.6904, 'grad_norm': 20.471603393554688, 'learning_rate': 8.313812483441879e-07, 'epoch': 0.1038784919989151, 'num_input_tokens_seen': 2480930816, 'completed': '32.09% (1_183 / 3_687)', 'remaining time': '21:47:09', 'throughput': '8109.04', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 18:48:17,928 >> {'loss': 0.3722, 'grad_norm': 14.457162857055664, 'learning_rate': 8.310661003304829e-07, 'epoch': 0.10414971521562245, 'num_input_tokens_seen': 2483027968, 'completed': '32.11% (1_184 / 3_687)', 'remaining time': '21:46:23', 'throughput': '9024.91', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 18:48:47,009 >> {'loss': 0.5091, 'grad_norm': 25.704435348510742, 'learning_rate': 8.30750726125476e-07, 'epoch': 0.1044209384323298, 'num_input_tokens_seen': 2485125120, 'completed': '32.14% (1_185 / 3_687)', 'remaining time': '21:45:37', 'throughput': '9014.35', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 18:49:15,388 >> {'loss': 0.4869, 'grad_norm': 16.720243453979492, 'learning_rate': 8.304351259829678e-07, 'epoch': 0.10469216164903716, 'num_input_tokens_seen': 2487222272, 'completed': '32.17% (1_186 / 3_687)', 'remaining time': '21:44:47', 'throughput': '9237.29', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 18:49:44,392 >> {'loss': 0.3771, 'grad_norm': 14.137613296508789, 'learning_rate': 8.301193001569418e-07, 'epoch': 0.10496338486574451, 'num_input_tokens_seen': 2489319424, 'completed': '32.19% (1_187 / 3_687)', 'remaining time': '21:44:01', 'throughput': '9038.20', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 18:50:15,412 >> {'loss': 0.317, 'grad_norm': 15.297533988952637, 'learning_rate': 8.298032489015623e-07, 'epoch': 0.10523460808245186, 'num_input_tokens_seen': 2491416576, 'completed': '32.22% (1_188 / 3_687)', 'remaining time': '21:43:28', 'throughput': '8450.61', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 18:50:44,660 >> {'loss': 0.5508, 'grad_norm': 21.42536163330078, 'learning_rate': 8.294869724711752e-07, 'epoch': 0.1055058312991592, 'num_input_tokens_seen': 2493513728, 'completed': '32.25% (1_189 / 3_687)', 'remaining time': '21:42:43', 'throughput': '8962.93', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 18:51:15,879 >> {'loss': 0.3826, 'grad_norm': 14.231220245361328, 'learning_rate': 8.291704711203082e-07, 'epoch': 0.10577705451586655, 'num_input_tokens_seen': 2495610880, 'completed': '32.28% (1_190 / 3_687)', 'remaining time': '21:42:12', 'throughput': '8396.99', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 18:51:48,578 >> {'loss': 0.3962, 'grad_norm': 16.597139358520508, 'learning_rate': 8.288537451036691e-07, 'epoch': 0.10604827773257391, 'num_input_tokens_seen': 2497708032, 'completed': '32.30% (1_191 / 3_687)', 'remaining time': '21:41:49', 'throughput': '8016.73', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 18:52:20,256 >> {'loss': 0.3347, 'grad_norm': 25.333345413208008, 'learning_rate': 8.28536794676147e-07, 'epoch': 0.10631950094928126, 'num_input_tokens_seen': 2499805184, 'completed': '32.33% (1_192 / 3_687)', 'remaining time': '21:41:20', 'throughput': '8275.45', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 18:52:53,673 >> {'loss': 0.4289, 'grad_norm': 29.860403060913086, 'learning_rate': 8.282196200928119e-07, 'epoch': 0.10659072416598861, 'num_input_tokens_seen': 2501902336, 'completed': '32.36% (1_193 / 3_687)', 'remaining time': '21:41:03', 'throughput': '7844.42', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 18:53:23,946 >> {'loss': 0.3256, 'grad_norm': 12.253214836120605, 'learning_rate': 8.279022216089135e-07, 'epoch': 0.10686194738269596, 'num_input_tokens_seen': 2503999488, 'completed': '32.38% (1_194 / 3_687)', 'remaining time': '21:40:25', 'throughput': '8659.37', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 18:53:55,064 >> {'loss': 0.4446, 'grad_norm': 17.277803421020508, 'learning_rate': 8.275845994798821e-07, 'epoch': 0.1071331705994033, 'num_input_tokens_seen': 2506096640, 'completed': '32.41% (1_195 / 3_687)', 'remaining time': '21:39:52', 'throughput': '8424.40', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 18:54:24,659 >> {'loss': 0.4053, 'grad_norm': 13.697084426879883, 'learning_rate': 8.272667539613281e-07, 'epoch': 0.10740439381611067, 'num_input_tokens_seen': 2508193792, 'completed': '32.44% (1_196 / 3_687)', 'remaining time': '21:39:10', 'throughput': '8857.56', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 18:54:56,223 >> {'loss': 0.7259, 'grad_norm': 22.16970443725586, 'learning_rate': 8.26948685309041e-07, 'epoch': 0.10767561703281801, 'num_input_tokens_seen': 2510290944, 'completed': '32.47% (1_197 / 3_687)', 'remaining time': '21:38:41', 'throughput': '8305.20', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 18:55:28,077 >> {'loss': 0.4267, 'grad_norm': 16.95806884765625, 'learning_rate': 8.266303937789908e-07, 'epoch': 0.10794684024952536, 'num_input_tokens_seen': 2512388096, 'completed': '32.49% (1_198 / 3_687)', 'remaining time': '21:38:13', 'throughput': '8229.52', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 18:55:57,582 >> {'loss': 0.5967, 'grad_norm': 15.100922584533691, 'learning_rate': 8.263118796273263e-07, 'epoch': 0.10821806346623271, 'num_input_tokens_seen': 2514485248, 'completed': '32.52% (1_199 / 3_687)', 'remaining time': '21:37:30', 'throughput': '8884.61', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 18:56:29,840 >> {'loss': 0.4512, 'grad_norm': 20.644742965698242, 'learning_rate': 8.259931431103754e-07, 'epoch': 0.10848928668294006, 'num_input_tokens_seen': 2516582400, 'completed': '32.55% (1_200 / 3_687)', 'remaining time': '21:37:05', 'throughput': '8126.53', 'gpu_mem_free': '5581MB'} +/scratch3/workspace/ctpham_umass_edu-ft/envs/prolong-final/lib/python3.10/site-packages/torch/distributed/fsdp/fully_sharded_data_parallel.py:689: FutureWarning: FSDP.state_dict_type() and FSDP.set_state_dict_type() are being deprecated. Please use APIs, get_state_dict() and set_state_dict(), which can support different parallelisms, FSDP1, FSDP2, DDP. API doc: https://pytorch.org/docs/stable/distributed.checkpoint.html#torch.distributed.checkpoint.state_dict.get_state_dict .Tutorial: https://pytorch.org/tutorials/recipes/distributed_checkpoint_recipe.html . + warnings.warn( +[INFO|trainer.py:3503] 2024-12-25 18:56:59,416 >> Saving model checkpoint to /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/checkpoint-1200 +[INFO|configuration_utils.py:472] 2024-12-25 18:56:59,419 >> Configuration saved in /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/checkpoint-1200/config.json +[INFO|configuration_utils.py:807] 2024-12-25 18:56:59,420 >> Configuration saved in /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/checkpoint-1200/generation_config.json +[INFO|modeling_utils.py:2807] 2024-12-25 18:58:06,272 >> The model is bigger than the maximum size per checkpoint (5GB) and is going to be split in 7 checkpoint shards. You can find where each parameters has been saved in the index located at /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/checkpoint-1200/model.safetensors.index.json. +[INFO|tokenization_utils_base.py:2684] 2024-12-25 18:58:06,276 >> tokenizer config file saved in /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/checkpoint-1200/tokenizer_config.json +[INFO|tokenization_utils_base.py:2693] 2024-12-25 18:58:06,277 >> Special tokens file saved in /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/checkpoint-1200/special_tokens_map.json +/scratch3/workspace/ctpham_umass_edu-ft/envs/prolong-final/lib/python3.10/site-packages/torch/distributed/fsdp/fully_sharded_data_parallel.py:689: FutureWarning: FSDP.state_dict_type() and FSDP.set_state_dict_type() are being deprecated. Please use APIs, get_state_dict() and set_state_dict(), which can support different parallelisms, FSDP1, FSDP2, DDP. API doc: https://pytorch.org/docs/stable/distributed.checkpoint.html#torch.distributed.checkpoint.state_dict.get_state_dict .Tutorial: https://pytorch.org/tutorials/recipes/distributed_checkpoint_recipe.html . + warnings.warn( +[WARNING|trainer.py:868] 2024-12-25 19:02:10,058 >> Save streaming dataset state: {'epoch': 0, 'sample_in_epoch': 2400, 'num_canonical_nodes': 1, 'shuffle_seed': 42, 'initial_physical_nodes': 1} +/scratch3/workspace/ctpham_umass_edu-ft/envs/prolong-final/lib/python3.10/site-packages/torch/utils/checkpoint.py:1399: FutureWarning: `torch.cpu.amp.autocast(args...)` is deprecated. Please use `torch.amp.autocast('cpu', args...)` instead. + with device_autocast_ctx, torch.cpu.amp.autocast(**cpu_autocast_kwargs), recompute_context: # type: ignore[attr-defined] +[INFO|trainer.py:175] 2024-12-25 19:02:41,669 >> {'loss': 0.5051, 'grad_norm': 16.619178771972656, 'learning_rate': 8.256741844846452e-07, 'epoch': 0.1087605098996474, 'num_input_tokens_seen': 2518679552, 'completed': '32.57% (1_201 / 3_687)', 'remaining time': '22:11:45', 'throughput': '705.01', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 19:03:16,491 >> {'loss': 0.3861, 'grad_norm': 13.504317283630371, 'learning_rate': 8.253550040068216e-07, 'epoch': 0.10903173311635476, 'num_input_tokens_seen': 2520776704, 'completed': '32.60% (1_202 / 3_687)', 'remaining time': '22:11:29', 'throughput': '7528.31', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 19:03:50,270 >> {'loss': 0.417, 'grad_norm': 15.428912162780762, 'learning_rate': 8.250356019337688e-07, 'epoch': 0.10930295633306211, 'num_input_tokens_seen': 2522873856, 'completed': '32.63% (1_203 / 3_687)', 'remaining time': '22:11:07', 'throughput': '7760.51', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 19:04:18,628 >> {'loss': 0.9491, 'grad_norm': 25.92603874206543, 'learning_rate': 8.247159785225295e-07, 'epoch': 0.10957417954976946, 'num_input_tokens_seen': 2524971008, 'completed': '32.66% (1_204 / 3_687)', 'remaining time': '22:10:12', 'throughput': '9244.11', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 19:04:46,436 >> {'loss': 0.756, 'grad_norm': 21.164356231689453, 'learning_rate': 8.243961340303245e-07, 'epoch': 0.10984540276647681, 'num_input_tokens_seen': 2527068160, 'completed': '32.68% (1_205 / 3_687)', 'remaining time': '22:09:13', 'throughput': '9426.88', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 19:05:16,744 >> {'loss': 0.2077, 'grad_norm': 9.049589157104492, 'learning_rate': 8.240760687145521e-07, 'epoch': 0.11011662598318415, 'num_input_tokens_seen': 2529165312, 'completed': '32.71% (1_206 / 3_687)', 'remaining time': '22:08:30', 'throughput': '8649.24', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 19:05:47,045 >> {'loss': 0.6235, 'grad_norm': 17.34234046936035, 'learning_rate': 8.237557828327891e-07, 'epoch': 0.11038784919989152, 'num_input_tokens_seen': 2531262464, 'completed': '32.74% (1_207 / 3_687)', 'remaining time': '22:07:47', 'throughput': '8651.23', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 19:06:19,016 >> {'loss': 0.6889, 'grad_norm': 21.27731704711914, 'learning_rate': 8.234352766427894e-07, 'epoch': 0.11065907241659886, 'num_input_tokens_seen': 2533359616, 'completed': '32.76% (1_208 / 3_687)', 'remaining time': '22:07:14', 'throughput': '8199.55', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 19:06:48,570 >> {'loss': 0.7666, 'grad_norm': 21.866683959960938, 'learning_rate': 8.231145504024838e-07, 'epoch': 0.11093029563330621, 'num_input_tokens_seen': 2535456768, 'completed': '32.79% (1_209 / 3_687)', 'remaining time': '22:06:26', 'throughput': '8870.01', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 19:07:19,275 >> {'loss': 0.3829, 'grad_norm': 12.252056121826172, 'learning_rate': 8.22793604369981e-07, 'epoch': 0.11120151885001356, 'num_input_tokens_seen': 2537553920, 'completed': '32.82% (1_210 / 3_687)', 'remaining time': '22:05:45', 'throughput': '8537.48', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 19:07:49,979 >> {'loss': 0.4294, 'grad_norm': 18.94308853149414, 'learning_rate': 8.224724388035659e-07, 'epoch': 0.1114727420667209, 'num_input_tokens_seen': 2539651072, 'completed': '32.85% (1_211 / 3_687)', 'remaining time': '22:05:05', 'throughput': '8537.88', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 19:08:19,971 >> {'loss': 0.5497, 'grad_norm': 18.54946517944336, 'learning_rate': 8.221510539617003e-07, 'epoch': 0.11174396528342827, 'num_input_tokens_seen': 2541748224, 'completed': '32.87% (1_212 / 3_687)', 'remaining time': '22:04:20', 'throughput': '8740.49', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 19:08:51,112 >> {'loss': 0.3906, 'grad_norm': 13.319576263427734, 'learning_rate': 8.218294501030226e-07, 'epoch': 0.11201518850013562, 'num_input_tokens_seen': 2543845376, 'completed': '32.90% (1_213 / 3_687)', 'remaining time': '22:03:42', 'throughput': '8417.82', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 19:09:23,222 >> {'loss': 0.4973, 'grad_norm': 14.632641792297363, 'learning_rate': 8.215076274863476e-07, 'epoch': 0.11228641171684296, 'num_input_tokens_seen': 2545942528, 'completed': '32.93% (1_214 / 3_687)', 'remaining time': '22:03:10', 'throughput': '8163.99', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 19:09:55,108 >> {'loss': 0.3886, 'grad_norm': 14.780923843383789, 'learning_rate': 8.211855863706654e-07, 'epoch': 0.11255763493355031, 'num_input_tokens_seen': 2548039680, 'completed': '32.95% (1_215 / 3_687)', 'remaining time': '22:02:36', 'throughput': '8221.35', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 19:10:26,172 >> {'loss': 0.5352, 'grad_norm': 17.888994216918945, 'learning_rate': 8.208633270151426e-07, 'epoch': 0.11282885815025766, 'num_input_tokens_seen': 2550136832, 'completed': '32.98% (1_216 / 3_687)', 'remaining time': '22:01:58', 'throughput': '8438.82', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 19:10:57,408 >> {'loss': 0.6396, 'grad_norm': 21.13353157043457, 'learning_rate': 8.205408496791216e-07, 'epoch': 0.11310008136696502, 'num_input_tokens_seen': 2552233984, 'completed': '33.01% (1_217 / 3_687)', 'remaining time': '22:01:21', 'throughput': '8392.39', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 19:11:31,901 >> {'loss': 0.3852, 'grad_norm': 15.377924919128418, 'learning_rate': 8.202181546221193e-07, 'epoch': 0.11337130458367237, 'num_input_tokens_seen': 2554331136, 'completed': '33.03% (1_218 / 3_687)', 'remaining time': '22:01:03', 'throughput': '7599.94', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 19:12:00,080 >> {'loss': 0.7451, 'grad_norm': 19.80136489868164, 'learning_rate': 8.19895242103829e-07, 'epoch': 0.11364252780037971, 'num_input_tokens_seen': 2556428288, 'completed': '33.06% (1_219 / 3_687)', 'remaining time': '22:00:08', 'throughput': '9302.83', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 19:12:29,122 >> {'loss': 0.7119, 'grad_norm': 20.684595108032227, 'learning_rate': 8.19572112384118e-07, 'epoch': 0.11391375101708706, 'num_input_tokens_seen': 2558525440, 'completed': '33.09% (1_220 / 3_687)', 'remaining time': '21:59:18', 'throughput': '9026.17', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 19:13:00,654 >> {'loss': 0.3151, 'grad_norm': 14.285638809204102, 'learning_rate': 8.192487657230288e-07, 'epoch': 0.11418497423379441, 'num_input_tokens_seen': 2560622592, 'completed': '33.12% (1_221 / 3_687)', 'remaining time': '21:58:42', 'throughput': '8313.59', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 19:13:34,484 >> {'loss': 0.3626, 'grad_norm': 15.149106979370117, 'learning_rate': 8.18925202380779e-07, 'epoch': 0.11445619745050176, 'num_input_tokens_seen': 2562719744, 'completed': '33.14% (1_222 / 3_687)', 'remaining time': '21:58:21', 'throughput': '7748.87', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 19:14:07,442 >> {'loss': 0.4939, 'grad_norm': 16.11283302307129, 'learning_rate': 8.186014226177594e-07, 'epoch': 0.11472742066720912, 'num_input_tokens_seen': 2564816896, 'completed': '33.17% (1_223 / 3_687)', 'remaining time': '21:57:54', 'throughput': '7953.98', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 19:14:37,126 >> {'loss': 0.4951, 'grad_norm': 16.55274200439453, 'learning_rate': 8.18277426694536e-07, 'epoch': 0.11499864388391647, 'num_input_tokens_seen': 2566914048, 'completed': '33.20% (1_224 / 3_687)', 'remaining time': '21:57:07', 'throughput': '8831.18', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 19:15:07,555 >> {'loss': 0.4981, 'grad_norm': 16.552610397338867, 'learning_rate': 8.179532148718483e-07, 'epoch': 0.11526986710062381, 'num_input_tokens_seen': 2569011200, 'completed': '33.22% (1_225 / 3_687)', 'remaining time': '21:56:26', 'throughput': '8614.85', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 19:15:38,815 >> {'loss': 0.6232, 'grad_norm': 20.186845779418945, 'learning_rate': 8.176287874106097e-07, 'epoch': 0.11554109031733116, 'num_input_tokens_seen': 2571108352, 'completed': '33.25% (1_226 / 3_687)', 'remaining time': '21:55:49', 'throughput': '8385.92', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 19:16:06,750 >> {'loss': 0.7157, 'grad_norm': 18.383283615112305, 'learning_rate': 8.173041445719069e-07, 'epoch': 0.11581231353403851, 'num_input_tokens_seen': 2573205504, 'completed': '33.28% (1_227 / 3_687)', 'remaining time': '21:54:53', 'throughput': '9384.15', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 19:16:38,856 >> {'loss': 0.7858, 'grad_norm': 28.215744018554688, 'learning_rate': 8.169792866170003e-07, 'epoch': 0.11608353675074587, 'num_input_tokens_seen': 2575302656, 'completed': '33.31% (1_228 / 3_687)', 'remaining time': '21:54:21', 'throughput': '8164.94', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 19:17:08,168 >> {'loss': 0.5306, 'grad_norm': 15.474285125732422, 'learning_rate': 8.166542138073232e-07, 'epoch': 0.11635475996745322, 'num_input_tokens_seen': 2577399808, 'completed': '33.33% (1_229 / 3_687)', 'remaining time': '21:53:33', 'throughput': '8943.09', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 19:17:40,750 >> {'loss': 0.5879, 'grad_norm': 30.002483367919922, 'learning_rate': 8.163289264044817e-07, 'epoch': 0.11662598318416056, 'num_input_tokens_seen': 2579496960, 'completed': '33.36% (1_230 / 3_687)', 'remaining time': '21:53:04', 'throughput': '8045.63', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 19:18:12,798 >> {'loss': 0.5828, 'grad_norm': 17.683021545410156, 'learning_rate': 8.160034246702548e-07, 'epoch': 0.11689720640086791, 'num_input_tokens_seen': 2581594112, 'completed': '33.39% (1_231 / 3_687)', 'remaining time': '21:52:32', 'throughput': '8179.92', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 19:18:44,396 >> {'loss': 0.4591, 'grad_norm': 18.124019622802734, 'learning_rate': 8.156777088665939e-07, 'epoch': 0.11716842961757526, 'num_input_tokens_seen': 2583691264, 'completed': '33.41% (1_232 / 3_687)', 'remaining time': '21:51:57', 'throughput': '8296.26', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 19:19:15,406 >> {'loss': 0.2872, 'grad_norm': 11.012289047241211, 'learning_rate': 8.153517792556226e-07, 'epoch': 0.11743965283428262, 'num_input_tokens_seen': 2585788416, 'completed': '33.44% (1_233 / 3_687)', 'remaining time': '21:51:19', 'throughput': '8453.52', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 19:19:44,060 >> {'loss': 0.6456, 'grad_norm': 19.892656326293945, 'learning_rate': 8.15025636099637e-07, 'epoch': 0.11771087605098997, 'num_input_tokens_seen': 2587885568, 'completed': '33.47% (1_234 / 3_687)', 'remaining time': '21:50:28', 'throughput': '9148.40', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 19:20:17,372 >> {'loss': 0.3137, 'grad_norm': 14.017526626586914, 'learning_rate': 8.146992796611042e-07, 'epoch': 0.11798209926769732, 'num_input_tokens_seen': 2589982720, 'completed': '33.50% (1_235 / 3_687)', 'remaining time': '21:50:03', 'throughput': '7869.44', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 19:20:44,383 >> {'loss': 0.8757, 'grad_norm': 21.26898765563965, 'learning_rate': 8.143727102026638e-07, 'epoch': 0.11825332248440466, 'num_input_tokens_seen': 2592079872, 'completed': '33.52% (1_236 / 3_687)', 'remaining time': '21:49:03', 'throughput': '9704.93', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 19:21:15,259 >> {'loss': 0.3173, 'grad_norm': 10.634222030639648, 'learning_rate': 8.140459279871264e-07, 'epoch': 0.11852454570111201, 'num_input_tokens_seen': 2594177024, 'completed': '33.55% (1_237 / 3_687)', 'remaining time': '21:48:24', 'throughput': '8490.43', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 19:21:45,877 >> {'loss': 0.7255, 'grad_norm': 20.876935958862305, 'learning_rate': 8.137189332774738e-07, 'epoch': 0.11879576891781937, 'num_input_tokens_seen': 2596274176, 'completed': '33.58% (1_238 / 3_687)', 'remaining time': '21:47:44', 'throughput': '8561.69', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 19:22:16,163 >> {'loss': 0.3449, 'grad_norm': 13.55611801147461, 'learning_rate': 8.133917263368589e-07, 'epoch': 0.11906699213452672, 'num_input_tokens_seen': 2598371328, 'completed': '33.60% (1_239 / 3_687)', 'remaining time': '21:47:02', 'throughput': '8655.68', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 19:22:47,340 >> {'loss': 0.4658, 'grad_norm': 23.81563949584961, 'learning_rate': 8.130643074286056e-07, 'epoch': 0.11933821535123407, 'num_input_tokens_seen': 2600468480, 'completed': '33.63% (1_240 / 3_687)', 'remaining time': '21:46:25', 'throughput': '8408.07', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 19:23:23,962 >> {'loss': 0.4361, 'grad_norm': 15.136983871459961, 'learning_rate': 8.127366768162077e-07, 'epoch': 0.11960943856794141, 'num_input_tokens_seen': 2602565632, 'completed': '33.66% (1_241 / 3_687)', 'remaining time': '21:46:19', 'throughput': '7158.22', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 19:23:56,246 >> {'loss': 0.3938, 'grad_norm': 14.615781784057617, 'learning_rate': 8.124088347633304e-07, 'epoch': 0.11988066178464876, 'num_input_tokens_seen': 2604662784, 'completed': '33.69% (1_242 / 3_687)', 'remaining time': '21:45:48', 'throughput': '8119.76', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 19:24:26,910 >> {'loss': 0.377, 'grad_norm': 13.33740234375, 'learning_rate': 8.120807815338083e-07, 'epoch': 0.12015188500135611, 'num_input_tokens_seen': 2606759936, 'completed': '33.71% (1_243 / 3_687)', 'remaining time': '21:45:08', 'throughput': '8548.88', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 19:25:00,896 >> {'loss': 0.3627, 'grad_norm': 16.450172424316406, 'learning_rate': 8.11752517391646e-07, 'epoch': 0.12042310821806347, 'num_input_tokens_seen': 2608857088, 'completed': '33.74% (1_244 / 3_687)', 'remaining time': '21:44:47', 'throughput': '7713.30', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 19:25:32,140 >> {'loss': 0.3407, 'grad_norm': 12.913908958435059, 'learning_rate': 8.114240426010183e-07, 'epoch': 0.12069433143477082, 'num_input_tokens_seen': 2610954240, 'completed': '33.77% (1_245 / 3_687)', 'remaining time': '21:44:11', 'throughput': '8390.46', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 19:26:01,656 >> {'loss': 0.6169, 'grad_norm': 19.959516525268555, 'learning_rate': 8.11095357426269e-07, 'epoch': 0.12096555465147817, 'num_input_tokens_seen': 2613051392, 'completed': '33.79% (1_246 / 3_687)', 'remaining time': '21:43:25', 'throughput': '8881.20', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 19:26:31,762 >> {'loss': 0.5242, 'grad_norm': 15.97438907623291, 'learning_rate': 8.107664621319113e-07, 'epoch': 0.12123677786818551, 'num_input_tokens_seen': 2615148544, 'completed': '33.82% (1_247 / 3_687)', 'remaining time': '21:42:42', 'throughput': '8707.37', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 19:27:01,492 >> {'loss': 0.3952, 'grad_norm': 14.837607383728027, 'learning_rate': 8.10437356982628e-07, 'epoch': 0.12150800108489286, 'num_input_tokens_seen': 2617245696, 'completed': '33.85% (1_248 / 3_687)', 'remaining time': '21:41:57', 'throughput': '8817.64', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 19:27:35,868 >> {'loss': 0.4423, 'grad_norm': 15.677940368652344, 'learning_rate': 8.1010804224327e-07, 'epoch': 0.12177922430160022, 'num_input_tokens_seen': 2619342848, 'completed': '33.88% (1_249 / 3_687)', 'remaining time': '21:41:38', 'throughput': '7625.74', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 19:28:05,735 >> {'loss': 0.3551, 'grad_norm': 13.296334266662598, 'learning_rate': 8.097785181788574e-07, 'epoch': 0.12205044751830757, 'num_input_tokens_seen': 2621440000, 'completed': '33.90% (1_250 / 3_687)', 'remaining time': '21:40:54', 'throughput': '8777.03', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 19:28:37,113 >> {'loss': 0.371, 'grad_norm': 17.426509857177734, 'learning_rate': 8.09448785054579e-07, 'epoch': 0.12232167073501492, 'num_input_tokens_seen': 2623537152, 'completed': '33.93% (1_251 / 3_687)', 'remaining time': '21:40:19', 'throughput': '8354.35', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 19:29:05,840 >> {'loss': 0.9746, 'grad_norm': 24.391117095947266, 'learning_rate': 8.091188431357908e-07, 'epoch': 0.12259289395172226, 'num_input_tokens_seen': 2625634304, 'completed': '33.96% (1_252 / 3_687)', 'remaining time': '21:39:29', 'throughput': '9125.50', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 19:29:35,732 >> {'loss': 0.9044, 'grad_norm': 22.590383529663086, 'learning_rate': 8.087886926880181e-07, 'epoch': 0.12286411716842961, 'num_input_tokens_seen': 2627731456, 'completed': '33.98% (1_253 / 3_687)', 'remaining time': '21:38:46', 'throughput': '8769.75', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 19:30:09,227 >> {'loss': 0.4665, 'grad_norm': 15.536231994628906, 'learning_rate': 8.084583339769531e-07, 'epoch': 0.12313534038513697, 'num_input_tokens_seen': 2629828608, 'completed': '34.01% (1_254 / 3_687)', 'remaining time': '21:38:22', 'throughput': '7826.30', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 19:30:40,307 >> {'loss': 0.3953, 'grad_norm': 16.86744499206543, 'learning_rate': 8.081277672684557e-07, 'epoch': 0.12340656360184432, 'num_input_tokens_seen': 2631925760, 'completed': '34.04% (1_255 / 3_687)', 'remaining time': '21:37:45', 'throughput': '8434.57', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 19:31:08,724 >> {'loss': 0.5952, 'grad_norm': 18.4736385345459, 'learning_rate': 8.077969928285541e-07, 'epoch': 0.12367778681855167, 'num_input_tokens_seen': 2634022912, 'completed': '34.07% (1_256 / 3_687)', 'remaining time': '21:36:53', 'throughput': '9224.69', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 19:31:40,027 >> {'loss': 0.4361, 'grad_norm': 15.2562837600708, 'learning_rate': 8.074660109234424e-07, 'epoch': 0.12394901003525902, 'num_input_tokens_seen': 2636120064, 'completed': '34.09% (1_257 / 3_687)', 'remaining time': '21:36:18', 'throughput': '8374.59', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 19:32:10,036 >> {'loss': 0.5753, 'grad_norm': 20.01945686340332, 'learning_rate': 8.071348218194823e-07, 'epoch': 0.12422023325196636, 'num_input_tokens_seen': 2638217216, 'completed': '34.12% (1_258 / 3_687)', 'remaining time': '21:35:35', 'throughput': '8735.49', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 19:32:41,080 >> {'loss': 0.7434, 'grad_norm': 20.903608322143555, 'learning_rate': 8.068034257832026e-07, 'epoch': 0.12449145646867373, 'num_input_tokens_seen': 2640314368, 'completed': '34.15% (1_259 / 3_687)', 'remaining time': '21:34:58', 'throughput': '8444.06', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 19:33:10,607 >> {'loss': 0.5717, 'grad_norm': 18.132099151611328, 'learning_rate': 8.064718230812976e-07, 'epoch': 0.12476267968538107, 'num_input_tokens_seen': 2642411520, 'completed': '34.17% (1_260 / 3_687)', 'remaining time': '21:34:13', 'throughput': '8878.31', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 19:33:42,033 >> {'loss': 0.6942, 'grad_norm': 19.513261795043945, 'learning_rate': 8.06140013980629e-07, 'epoch': 0.1250339029020884, 'num_input_tokens_seen': 2644508672, 'completed': '34.20% (1_261 / 3_687)', 'remaining time': '21:33:38', 'throughput': '8341.87', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 19:34:14,349 >> {'loss': 0.3526, 'grad_norm': 15.275543212890625, 'learning_rate': 8.05807998748224e-07, 'epoch': 0.12530512611879577, 'num_input_tokens_seen': 2646605824, 'completed': '34.23% (1_262 / 3_687)', 'remaining time': '21:33:07', 'throughput': '8111.70', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 19:34:44,683 >> {'loss': 0.3085, 'grad_norm': 15.771416664123535, 'learning_rate': 8.05475777651276e-07, 'epoch': 0.12557634933550313, 'num_input_tokens_seen': 2648702976, 'completed': '34.26% (1_263 / 3_687)', 'remaining time': '21:32:27', 'throughput': '8641.85', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 19:35:14,140 >> {'loss': 0.6849, 'grad_norm': 20.19382095336914, 'learning_rate': 8.051433509571435e-07, 'epoch': 0.12584757255221046, 'num_input_tokens_seen': 2650800128, 'completed': '34.28% (1_264 / 3_687)', 'remaining time': '21:31:42', 'throughput': '8899.26', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 19:35:46,718 >> {'loss': 0.5637, 'grad_norm': 19.399974822998047, 'learning_rate': 8.04810718933351e-07, 'epoch': 0.12611879576891782, 'num_input_tokens_seen': 2652897280, 'completed': '34.31% (1_265 / 3_687)', 'remaining time': '21:31:13', 'throughput': '8046.45', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 19:36:17,134 >> {'loss': 0.3837, 'grad_norm': 14.588866233825684, 'learning_rate': 8.044778818475884e-07, 'epoch': 0.12639001898562516, 'num_input_tokens_seen': 2654994432, 'completed': '34.34% (1_266 / 3_687)', 'remaining time': '21:30:33', 'throughput': '8618.62', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 19:36:47,465 >> {'loss': 0.6066, 'grad_norm': 18.231657028198242, 'learning_rate': 8.0414483996771e-07, 'epoch': 0.12666124220233252, 'num_input_tokens_seen': 2657091584, 'completed': '34.36% (1_267 / 3_687)', 'remaining time': '21:29:52', 'throughput': '8643.01', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 19:37:19,628 >> {'loss': 0.8268, 'grad_norm': 21.163400650024414, 'learning_rate': 8.038115935617355e-07, 'epoch': 0.12693246541903988, 'num_input_tokens_seen': 2659188736, 'completed': '34.39% (1_268 / 3_687)', 'remaining time': '21:29:21', 'throughput': '8150.42', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 19:37:49,592 >> {'loss': 0.3457, 'grad_norm': 13.187846183776855, 'learning_rate': 8.034781428978484e-07, 'epoch': 0.12720368863574721, 'num_input_tokens_seen': 2661285888, 'completed': '34.42% (1_269 / 3_687)', 'remaining time': '21:28:39', 'throughput': '8748.57', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 19:38:19,440 >> {'loss': 0.223, 'grad_norm': 11.001519203186035, 'learning_rate': 8.031444882443976e-07, 'epoch': 0.12747491185245458, 'num_input_tokens_seen': 2663383040, 'completed': '34.45% (1_270 / 3_687)', 'remaining time': '21:27:56', 'throughput': '8782.72', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 19:38:52,272 >> {'loss': 0.5616, 'grad_norm': 16.197965621948242, 'learning_rate': 8.028106298698957e-07, 'epoch': 0.1277461350691619, 'num_input_tokens_seen': 2665480192, 'completed': '34.47% (1_271 / 3_687)', 'remaining time': '21:27:28', 'throughput': '7984.30', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 19:39:22,906 >> {'loss': 0.314, 'grad_norm': 13.498997688293457, 'learning_rate': 8.024765680430188e-07, 'epoch': 0.12801735828586927, 'num_input_tokens_seen': 2667577344, 'completed': '34.50% (1_272 / 3_687)', 'remaining time': '21:26:49', 'throughput': '8557.41', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 19:39:54,178 >> {'loss': 0.585, 'grad_norm': 18.05949592590332, 'learning_rate': 8.021423030326075e-07, 'epoch': 0.12828858150257663, 'num_input_tokens_seen': 2669674496, 'completed': '34.53% (1_273 / 3_687)', 'remaining time': '21:26:14', 'throughput': '8382.72', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 19:40:24,034 >> {'loss': 0.2417, 'grad_norm': 10.726323127746582, 'learning_rate': 8.018078351076653e-07, 'epoch': 0.12855980471928397, 'num_input_tokens_seen': 2671771648, 'completed': '34.55% (1_274 / 3_687)', 'remaining time': '21:25:31', 'throughput': '8780.36', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 19:40:57,426 >> {'loss': 0.3998, 'grad_norm': 13.169657707214355, 'learning_rate': 8.014731645373595e-07, 'epoch': 0.12883102793599133, 'num_input_tokens_seen': 2673868800, 'completed': '34.58% (1_275 / 3_687)', 'remaining time': '21:25:06', 'throughput': '7850.28', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 19:41:29,315 >> {'loss': 0.5083, 'grad_norm': 18.775423049926758, 'learning_rate': 8.011382915910203e-07, 'epoch': 0.12910225115269866, 'num_input_tokens_seen': 2675965952, 'completed': '34.61% (1_276 / 3_687)', 'remaining time': '21:24:34', 'throughput': '8220.57', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 19:42:00,640 >> {'loss': 0.324, 'grad_norm': 14.191473007202148, 'learning_rate': 8.008032165381403e-07, 'epoch': 0.12937347436940602, 'num_input_tokens_seen': 2678063104, 'completed': '34.64% (1_277 / 3_687)', 'remaining time': '21:23:59', 'throughput': '8368.54', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 19:42:31,640 >> {'loss': 0.2584, 'grad_norm': 19.03130531311035, 'learning_rate': 8.004679396483756e-07, 'epoch': 0.12964469758611338, 'num_input_tokens_seen': 2680160256, 'completed': '34.66% (1_278 / 3_687)', 'remaining time': '21:23:22', 'throughput': '8456.32', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 19:43:02,003 >> {'loss': 0.8237, 'grad_norm': 22.582111358642578, 'learning_rate': 8.001324611915441e-07, 'epoch': 0.12991592080282072, 'num_input_tokens_seen': 2682257408, 'completed': '34.69% (1_279 / 3_687)', 'remaining time': '21:22:42', 'throughput': '8633.74', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 19:43:31,739 >> {'loss': 0.4733, 'grad_norm': 16.168638229370117, 'learning_rate': 7.99796781437626e-07, 'epoch': 0.13018714401952808, 'num_input_tokens_seen': 2684354560, 'completed': '34.72% (1_280 / 3_687)', 'remaining time': '21:21:59', 'throughput': '8815.74', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 19:44:06,936 >> {'loss': 0.2726, 'grad_norm': 11.249455451965332, 'learning_rate': 7.994609006567635e-07, 'epoch': 0.1304583672362354, 'num_input_tokens_seen': 2686451712, 'completed': '34.74% (1_281 / 3_687)', 'remaining time': '21:21:43', 'throughput': '7447.91', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 19:44:37,393 >> {'loss': 0.4514, 'grad_norm': 24.108003616333008, 'learning_rate': 7.99124819119261e-07, 'epoch': 0.13072959045294277, 'num_input_tokens_seen': 2688548864, 'completed': '34.77% (1_282 / 3_687)', 'remaining time': '21:21:03', 'throughput': '8606.97', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 19:45:05,834 >> {'loss': 0.6907, 'grad_norm': 22.541616439819336, 'learning_rate': 7.987885370955841e-07, 'epoch': 0.13100081366965013, 'num_input_tokens_seen': 2690646016, 'completed': '34.80% (1_283 / 3_687)', 'remaining time': '21:20:14', 'throughput': '9216.95', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 19:45:34,268 >> {'loss': 0.6375, 'grad_norm': 17.23971176147461, 'learning_rate': 7.984520548563594e-07, 'epoch': 0.13127203688635747, 'num_input_tokens_seen': 2692743168, 'completed': '34.83% (1_284 / 3_687)', 'remaining time': '21:19:25', 'throughput': '9219.33', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 19:46:05,282 >> {'loss': 0.4198, 'grad_norm': 14.531922340393066, 'learning_rate': 7.981153726723755e-07, 'epoch': 0.13154326010306483, 'num_input_tokens_seen': 2694840320, 'completed': '34.85% (1_285 / 3_687)', 'remaining time': '21:18:48', 'throughput': '8452.51', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 19:46:35,059 >> {'loss': 0.3334, 'grad_norm': 12.036006927490234, 'learning_rate': 7.977784908145809e-07, 'epoch': 0.13181448331977216, 'num_input_tokens_seen': 2696937472, 'completed': '34.88% (1_286 / 3_687)', 'remaining time': '21:18:05', 'throughput': '8803.61', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 19:47:06,921 >> {'loss': 0.2007, 'grad_norm': 10.315123558044434, 'learning_rate': 7.974414095540858e-07, 'epoch': 0.13208570653647952, 'num_input_tokens_seen': 2699034624, 'completed': '34.91% (1_287 / 3_687)', 'remaining time': '21:17:33', 'throughput': '8227.44', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 19:47:37,720 >> {'loss': 0.3, 'grad_norm': 10.276031494140625, 'learning_rate': 7.971041291621601e-07, 'epoch': 0.13235692975318689, 'num_input_tokens_seen': 2701131776, 'completed': '34.93% (1_288 / 3_687)', 'remaining time': '21:16:56', 'throughput': '8511.57', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 19:48:13,104 >> {'loss': 0.4751, 'grad_norm': 16.799928665161133, 'learning_rate': 7.967666499102341e-07, 'epoch': 0.13262815296989422, 'num_input_tokens_seen': 2703228928, 'completed': '34.96% (1_289 / 3_687)', 'remaining time': '21:16:40', 'throughput': '7408.50', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 19:48:44,921 >> {'loss': 0.3534, 'grad_norm': 13.778836250305176, 'learning_rate': 7.964289720698986e-07, 'epoch': 0.13289937618660158, 'num_input_tokens_seen': 2705326080, 'completed': '34.99% (1_290 / 3_687)', 'remaining time': '21:16:08', 'throughput': '8239.21', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 19:49:18,180 >> {'loss': 0.6054, 'grad_norm': 17.865440368652344, 'learning_rate': 7.960910959129037e-07, 'epoch': 0.13317059940330891, 'num_input_tokens_seen': 2707423232, 'completed': '35.01% (1_291 / 3_687)', 'remaining time': '21:15:42', 'throughput': '7881.72', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 19:49:47,631 >> {'loss': 0.5136, 'grad_norm': 17.34958267211914, 'learning_rate': 7.957530217111591e-07, 'epoch': 0.13344182262001628, 'num_input_tokens_seen': 2709520384, 'completed': '35.04% (1_292 / 3_687)', 'remaining time': '21:14:58', 'throughput': '8901.26', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 19:50:17,946 >> {'loss': 0.853, 'grad_norm': 24.52153205871582, 'learning_rate': 7.954147497367343e-07, 'epoch': 0.13371304583672364, 'num_input_tokens_seen': 2711617536, 'completed': '35.07% (1_293 / 3_687)', 'remaining time': '21:14:18', 'throughput': '8647.24', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 19:50:47,195 >> {'loss': 0.4762, 'grad_norm': 18.36350440979004, 'learning_rate': 7.950762802618576e-07, 'epoch': 0.13398426905343097, 'num_input_tokens_seen': 2713714688, 'completed': '35.10% (1_294 / 3_687)', 'remaining time': '21:13:34', 'throughput': '8962.43', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 19:51:15,323 >> {'loss': 0.5823, 'grad_norm': 17.231550216674805, 'learning_rate': 7.947376135589164e-07, 'epoch': 0.13425549227013833, 'num_input_tokens_seen': 2715811840, 'completed': '35.12% (1_295 / 3_687)', 'remaining time': '21:12:43', 'throughput': '9319.68', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 19:51:45,055 >> {'loss': 0.7359, 'grad_norm': 22.24859619140625, 'learning_rate': 7.943987499004569e-07, 'epoch': 0.13452671548684567, 'num_input_tokens_seen': 2717908992, 'completed': '35.15% (1_296 / 3_687)', 'remaining time': '21:12:01', 'throughput': '8817.02', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 19:52:14,273 >> {'loss': 0.6906, 'grad_norm': 22.393712997436523, 'learning_rate': 7.940596895591835e-07, 'epoch': 0.13479793870355303, 'num_input_tokens_seen': 2720006144, 'completed': '35.18% (1_297 / 3_687)', 'remaining time': '21:11:16', 'throughput': '8972.07', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 19:52:45,089 >> {'loss': 0.3609, 'grad_norm': 13.456273078918457, 'learning_rate': 7.93720432807959e-07, 'epoch': 0.13506916192026036, 'num_input_tokens_seen': 2722103296, 'completed': '35.20% (1_298 / 3_687)', 'remaining time': '21:10:39', 'throughput': '8506.58', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 19:53:14,456 >> {'loss': 0.5547, 'grad_norm': 18.195858001708984, 'learning_rate': 7.933809799198045e-07, 'epoch': 0.13534038513696772, 'num_input_tokens_seen': 2724200448, 'completed': '35.23% (1_299 / 3_687)', 'remaining time': '21:09:55', 'throughput': '8926.39', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 19:53:45,111 >> {'loss': 0.4255, 'grad_norm': 23.925018310546875, 'learning_rate': 7.930413311678987e-07, 'epoch': 0.13561160835367508, 'num_input_tokens_seen': 2726297600, 'completed': '35.26% (1_300 / 3_687)', 'remaining time': '21:09:17', 'throughput': '8551.61', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 19:54:15,184 >> {'loss': 0.3462, 'grad_norm': 17.64158821105957, 'learning_rate': 7.92701486825578e-07, 'epoch': 0.13588283157038242, 'num_input_tokens_seen': 2728394752, 'completed': '35.29% (1_301 / 3_687)', 'remaining time': '21:08:36', 'throughput': '8716.96', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 19:54:49,236 >> {'loss': 0.5714, 'grad_norm': 25.17290496826172, 'learning_rate': 7.923614471663361e-07, 'epoch': 0.13615405478708978, 'num_input_tokens_seen': 2730491904, 'completed': '35.31% (1_302 / 3_687)', 'remaining time': '21:08:14', 'throughput': '7698.28', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 19:55:18,419 >> {'loss': 0.2677, 'grad_norm': 10.592412948608398, 'learning_rate': 7.920212124638241e-07, 'epoch': 0.1364252780037971, 'num_input_tokens_seen': 2732589056, 'completed': '35.34% (1_303 / 3_687)', 'remaining time': '21:07:29', 'throughput': '8982.82', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 19:55:48,508 >> {'loss': 0.3103, 'grad_norm': 12.378978729248047, 'learning_rate': 7.916807829918499e-07, 'epoch': 0.13669650122050447, 'num_input_tokens_seen': 2734686208, 'completed': '35.37% (1_304 / 3_687)', 'remaining time': '21:06:49', 'throughput': '8712.32', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 19:56:18,337 >> {'loss': 0.3225, 'grad_norm': 17.801116943359375, 'learning_rate': 7.913401590243781e-07, 'epoch': 0.13696772443721184, 'num_input_tokens_seen': 2736783360, 'completed': '35.39% (1_305 / 3_687)', 'remaining time': '21:06:07', 'throughput': '8788.23', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 19:56:50,131 >> {'loss': 0.5376, 'grad_norm': 19.611040115356445, 'learning_rate': 7.909993408355302e-07, 'epoch': 0.13723894765391917, 'num_input_tokens_seen': 2738880512, 'completed': '35.42% (1_306 / 3_687)', 'remaining time': '21:05:35', 'throughput': '8244.91', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 19:57:19,154 >> {'loss': 0.644, 'grad_norm': 20.099191665649414, 'learning_rate': 7.906583286995834e-07, 'epoch': 0.13751017087062653, 'num_input_tokens_seen': 2740977664, 'completed': '35.45% (1_307 / 3_687)', 'remaining time': '21:04:50', 'throughput': '9032.46', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 19:57:48,426 >> {'loss': 0.2898, 'grad_norm': 13.194962501525879, 'learning_rate': 7.903171228909714e-07, 'epoch': 0.13778139408733386, 'num_input_tokens_seen': 2743074816, 'completed': '35.48% (1_308 / 3_687)', 'remaining time': '21:04:06', 'throughput': '8955.42', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 19:58:22,255 >> {'loss': 0.3795, 'grad_norm': 13.878562927246094, 'learning_rate': 7.899757236842836e-07, 'epoch': 0.13805261730404123, 'num_input_tokens_seen': 2745171968, 'completed': '35.50% (1_309 / 3_687)', 'remaining time': '21:03:43', 'throughput': '7748.95', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 19:58:53,669 >> {'loss': 0.7278, 'grad_norm': 25.963001251220703, 'learning_rate': 7.89634131354265e-07, 'epoch': 0.1383238405207486, 'num_input_tokens_seen': 2747269120, 'completed': '35.53% (1_310 / 3_687)', 'remaining time': '21:03:09', 'throughput': '8345.01', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 19:59:23,995 >> {'loss': 0.5589, 'grad_norm': 19.090431213378906, 'learning_rate': 7.892923461758165e-07, 'epoch': 0.13859506373745592, 'num_input_tokens_seen': 2749366272, 'completed': '35.56% (1_311 / 3_687)', 'remaining time': '21:02:30', 'throughput': '8644.11', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 19:59:53,855 >> {'loss': 0.4635, 'grad_norm': 15.916430473327637, 'learning_rate': 7.889503684239933e-07, 'epoch': 0.13886628695416328, 'num_input_tokens_seen': 2751463424, 'completed': '35.58% (1_312 / 3_687)', 'remaining time': '21:01:48', 'throughput': '8779.11', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 20:00:24,348 >> {'loss': 0.3096, 'grad_norm': 12.155187606811523, 'learning_rate': 7.886081983740066e-07, 'epoch': 0.13913751017087062, 'num_input_tokens_seen': 2753560576, 'completed': '35.61% (1_313 / 3_687)', 'remaining time': '21:01:10', 'throughput': '8596.77', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 20:00:53,207 >> {'loss': 0.5558, 'grad_norm': 19.226377487182617, 'learning_rate': 7.882658363012214e-07, 'epoch': 0.13940873338757798, 'num_input_tokens_seen': 2755657728, 'completed': '35.64% (1_314 / 3_687)', 'remaining time': '21:00:24', 'throughput': '9083.81', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 20:01:23,482 >> {'loss': 0.4074, 'grad_norm': 16.780195236206055, 'learning_rate': 7.879232824811579e-07, 'epoch': 0.13967995660428534, 'num_input_tokens_seen': 2757754880, 'completed': '35.67% (1_315 / 3_687)', 'remaining time': '20:59:45', 'throughput': '8658.73', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 20:01:52,315 >> {'loss': 0.8599, 'grad_norm': 24.132198333740234, 'learning_rate': 7.875805371894904e-07, 'epoch': 0.13995117982099267, 'num_input_tokens_seen': 2759852032, 'completed': '35.69% (1_316 / 3_687)', 'remaining time': '20:58:59', 'throughput': '9091.76', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 20:02:25,130 >> {'loss': 0.4509, 'grad_norm': 16.48426055908203, 'learning_rate': 7.872376007020469e-07, 'epoch': 0.14022240303770003, 'num_input_tokens_seen': 2761949184, 'completed': '35.72% (1_317 / 3_687)', 'remaining time': '20:58:32', 'throughput': '7988.51', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 20:02:56,615 >> {'loss': 0.3677, 'grad_norm': 13.625266075134277, 'learning_rate': 7.8689447329481e-07, 'epoch': 0.14049362625440737, 'num_input_tokens_seen': 2764046336, 'completed': '35.75% (1_318 / 3_687)', 'remaining time': '20:57:58', 'throughput': '8326.08', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 20:03:29,766 >> {'loss': 0.5272, 'grad_norm': 17.589357376098633, 'learning_rate': 7.865511552439156e-07, 'epoch': 0.14076484947111473, 'num_input_tokens_seen': 2766143488, 'completed': '35.77% (1_319 / 3_687)', 'remaining time': '20:57:32', 'throughput': '7907.40', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 20:03:57,394 >> {'loss': 0.5088, 'grad_norm': 15.197576522827148, 'learning_rate': 7.862076468256529e-07, 'epoch': 0.1410360726878221, 'num_input_tokens_seen': 2768240640, 'completed': '35.80% (1_320 / 3_687)', 'remaining time': '20:56:41', 'throughput': '9488.36', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 20:04:28,026 >> {'loss': 0.4647, 'grad_norm': 18.043136596679688, 'learning_rate': 7.858639483164643e-07, 'epoch': 0.14130729590452942, 'num_input_tokens_seen': 2770337792, 'completed': '35.83% (1_321 / 3_687)', 'remaining time': '20:56:04', 'throughput': '8557.97', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 20:04:59,031 >> {'loss': 0.5039, 'grad_norm': 15.092884063720703, 'learning_rate': 7.855200599929456e-07, 'epoch': 0.14157851912123678, 'num_input_tokens_seen': 2772434944, 'completed': '35.86% (1_322 / 3_687)', 'remaining time': '20:55:28', 'throughput': '8455.02', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 20:05:29,581 >> {'loss': 0.424, 'grad_norm': 15.708367347717285, 'learning_rate': 7.851759821318447e-07, 'epoch': 0.14184974233794412, 'num_input_tokens_seen': 2774532096, 'completed': '35.88% (1_323 / 3_687)', 'remaining time': '20:54:50', 'throughput': '8580.66', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 20:05:59,982 >> {'loss': 0.5721, 'grad_norm': 22.44023323059082, 'learning_rate': 7.848317150100624e-07, 'epoch': 0.14212096555465148, 'num_input_tokens_seen': 2776629248, 'completed': '35.91% (1_324 / 3_687)', 'remaining time': '20:54:12', 'throughput': '8622.91', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 20:06:31,735 >> {'loss': 0.4114, 'grad_norm': 13.735273361206055, 'learning_rate': 7.84487258904652e-07, 'epoch': 0.14239218877135884, 'num_input_tokens_seen': 2778726400, 'completed': '35.94% (1_325 / 3_687)', 'remaining time': '20:53:40', 'throughput': '8255.84', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 20:07:02,168 >> {'loss': 0.7805, 'grad_norm': 24.59619140625, 'learning_rate': 7.841426140928184e-07, 'epoch': 0.14266341198806617, 'num_input_tokens_seen': 2780823552, 'completed': '35.96% (1_326 / 3_687)', 'remaining time': '20:53:01', 'throughput': '8613.59', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 20:07:34,994 >> {'loss': 0.7438, 'grad_norm': 24.768539428710938, 'learning_rate': 7.837977808519189e-07, 'epoch': 0.14293463520477354, 'num_input_tokens_seen': 2782920704, 'completed': '35.99% (1_327 / 3_687)', 'remaining time': '20:52:34', 'throughput': '7986.00', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 20:08:06,338 >> {'loss': 0.6571, 'grad_norm': 20.256254196166992, 'learning_rate': 7.834527594594618e-07, 'epoch': 0.14320585842148087, 'num_input_tokens_seen': 2785017856, 'completed': '36.02% (1_328 / 3_687)', 'remaining time': '20:52:00', 'throughput': '8363.32', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 20:08:39,747 >> {'loss': 0.5158, 'grad_norm': 18.815889358520508, 'learning_rate': 7.831075501931078e-07, 'epoch': 0.14347708163818823, 'num_input_tokens_seen': 2787115008, 'completed': '36.05% (1_329 / 3_687)', 'remaining time': '20:51:35', 'throughput': '7846.51', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 20:09:06,962 >> {'loss': 0.6597, 'grad_norm': 18.49612045288086, 'learning_rate': 7.827621533306677e-07, 'epoch': 0.1437483048548956, 'num_input_tokens_seen': 2789212160, 'completed': '36.07% (1_330 / 3_687)', 'remaining time': '20:50:42', 'throughput': '9632.22', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 20:09:35,565 >> {'loss': 0.479, 'grad_norm': 15.946340560913086, 'learning_rate': 7.824165691501036e-07, 'epoch': 0.14401952807160293, 'num_input_tokens_seen': 2791309312, 'completed': '36.10% (1_331 / 3_687)', 'remaining time': '20:49:56', 'throughput': '9165.10', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 20:10:06,115 >> {'loss': 0.3821, 'grad_norm': 17.465255737304688, 'learning_rate': 7.820707979295289e-07, 'epoch': 0.1442907512883103, 'num_input_tokens_seen': 2793406464, 'completed': '36.13% (1_332 / 3_687)', 'remaining time': '20:49:19', 'throughput': '8580.86', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 20:10:37,905 >> {'loss': 0.6755, 'grad_norm': 28.799354553222656, 'learning_rate': 7.81724839947207e-07, 'epoch': 0.14456197450501762, 'num_input_tokens_seen': 2795503616, 'completed': '36.15% (1_333 / 3_687)', 'remaining time': '20:48:47', 'throughput': '8246.13', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 20:11:08,278 >> {'loss': 0.4634, 'grad_norm': 19.517778396606445, 'learning_rate': 7.813786954815517e-07, 'epoch': 0.14483319772172498, 'num_input_tokens_seen': 2797600768, 'completed': '36.18% (1_334 / 3_687)', 'remaining time': '20:48:09', 'throughput': '8630.63', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 20:11:38,952 >> {'loss': 0.3967, 'grad_norm': 18.770734786987305, 'learning_rate': 7.810323648111269e-07, 'epoch': 0.14510442093843234, 'num_input_tokens_seen': 2799697920, 'completed': '36.21% (1_335 / 3_687)', 'remaining time': '20:47:32', 'throughput': '8546.38', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 20:12:13,009 >> {'loss': 0.4468, 'grad_norm': 14.419391632080078, 'learning_rate': 7.806858482146463e-07, 'epoch': 0.14537564415513968, 'num_input_tokens_seen': 2801795072, 'completed': '36.24% (1_336 / 3_687)', 'remaining time': '20:47:10', 'throughput': '7697.14', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 20:12:44,412 >> {'loss': 0.4719, 'grad_norm': 17.39090919494629, 'learning_rate': 7.803391459709733e-07, 'epoch': 0.14564686737184704, 'num_input_tokens_seen': 2803892224, 'completed': '36.26% (1_337 / 3_687)', 'remaining time': '20:46:36', 'throughput': '8347.79', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 20:13:14,192 >> {'loss': 0.4681, 'grad_norm': 16.507200241088867, 'learning_rate': 7.799922583591206e-07, 'epoch': 0.14591809058855437, 'num_input_tokens_seen': 2805989376, 'completed': '36.29% (1_338 / 3_687)', 'remaining time': '20:45:55', 'throughput': '8802.64', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 20:13:44,357 >> {'loss': 0.4491, 'grad_norm': 14.502838134765625, 'learning_rate': 7.796451856582504e-07, 'epoch': 0.14618931380526173, 'num_input_tokens_seen': 2808086528, 'completed': '36.32% (1_339 / 3_687)', 'remaining time': '20:45:16', 'throughput': '8690.20', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 20:14:14,052 >> {'loss': 0.332, 'grad_norm': 12.314398765563965, 'learning_rate': 7.792979281476734e-07, 'epoch': 0.14646053702196907, 'num_input_tokens_seen': 2810183680, 'completed': '36.34% (1_340 / 3_687)', 'remaining time': '20:44:35', 'throughput': '8828.07', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 20:14:47,765 >> {'loss': 0.4365, 'grad_norm': 13.12981128692627, 'learning_rate': 7.789504861068492e-07, 'epoch': 0.14673176023867643, 'num_input_tokens_seen': 2812280832, 'completed': '36.37% (1_341 / 3_687)', 'remaining time': '20:44:11', 'throughput': '7775.64', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 20:15:16,930 >> {'loss': 0.516, 'grad_norm': 19.02878761291504, 'learning_rate': 7.78602859815386e-07, 'epoch': 0.1470029834553838, 'num_input_tokens_seen': 2814377984, 'completed': '36.40% (1_342 / 3_687)', 'remaining time': '20:43:28', 'throughput': '8988.22', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 20:15:46,348 >> {'loss': 0.2956, 'grad_norm': 11.80683422088623, 'learning_rate': 7.782550495530402e-07, 'epoch': 0.14727420667209112, 'num_input_tokens_seen': 2816475136, 'completed': '36.43% (1_343 / 3_687)', 'remaining time': '20:42:46', 'throughput': '8911.08', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 20:16:16,226 >> {'loss': 0.7255, 'grad_norm': 23.65553855895996, 'learning_rate': 7.779070555997162e-07, 'epoch': 0.14754542988879848, 'num_input_tokens_seen': 2818572288, 'completed': '36.45% (1_344 / 3_687)', 'remaining time': '20:42:06', 'throughput': '8774.01', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 20:16:48,696 >> {'loss': 0.6579, 'grad_norm': 20.80998420715332, 'learning_rate': 7.775588782354666e-07, 'epoch': 0.14781665310550582, 'num_input_tokens_seen': 2820669440, 'completed': '36.48% (1_345 / 3_687)', 'remaining time': '20:41:37', 'throughput': '8073.27', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 20:17:24,675 >> {'loss': 0.7579, 'grad_norm': 20.969602584838867, 'learning_rate': 7.77210517740491e-07, 'epoch': 0.14808787632221318, 'num_input_tokens_seen': 2822766592, 'completed': '36.51% (1_346 / 3_687)', 'remaining time': '20:41:23', 'throughput': '7285.96', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 20:17:56,212 >> {'loss': 0.2488, 'grad_norm': 9.614218711853027, 'learning_rate': 7.768619743951366e-07, 'epoch': 0.14835909953892054, 'num_input_tokens_seen': 2824863744, 'completed': '36.53% (1_347 / 3_687)', 'remaining time': '20:40:50', 'throughput': '8312.41', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 20:18:26,116 >> {'loss': 0.3422, 'grad_norm': 12.458992004394531, 'learning_rate': 7.765132484798978e-07, 'epoch': 0.14863032275562787, 'num_input_tokens_seen': 2826960896, 'completed': '36.56% (1_348 / 3_687)', 'remaining time': '20:40:10', 'throughput': '8766.08', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 20:18:56,658 >> {'loss': 0.5978, 'grad_norm': 31.408218383789062, 'learning_rate': 7.761643402754163e-07, 'epoch': 0.14890154597233524, 'num_input_tokens_seen': 2829058048, 'completed': '36.59% (1_349 / 3_687)', 'remaining time': '20:39:33', 'throughput': '8583.26', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 20:19:27,732 >> {'loss': 0.5435, 'grad_norm': 19.415218353271484, 'learning_rate': 7.7581525006248e-07, 'epoch': 0.14917276918904257, 'num_input_tokens_seen': 2831155200, 'completed': '36.62% (1_350 / 3_687)', 'remaining time': '20:38:58', 'throughput': '8435.94', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 20:20:01,980 >> {'loss': 0.4826, 'grad_norm': 17.554052352905273, 'learning_rate': 7.754659781220232e-07, 'epoch': 0.14944399240574993, 'num_input_tokens_seen': 2833252352, 'completed': '36.64% (1_351 / 3_687)', 'remaining time': '20:38:36', 'throughput': '7654.33', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 20:20:32,706 >> {'loss': 0.3339, 'grad_norm': 14.18742561340332, 'learning_rate': 7.751165247351269e-07, 'epoch': 0.1497152156224573, 'num_input_tokens_seen': 2835349504, 'completed': '36.67% (1_352 / 3_687)', 'remaining time': '20:38:00', 'throughput': '8531.58', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 20:21:03,123 >> {'loss': 0.5135, 'grad_norm': 15.89776611328125, 'learning_rate': 7.747668901830178e-07, 'epoch': 0.14998643883916463, 'num_input_tokens_seen': 2837446656, 'completed': '36.70% (1_353 / 3_687)', 'remaining time': '20:37:22', 'throughput': '8618.36', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 20:21:33,933 >> {'loss': 0.5667, 'grad_norm': 19.270442962646484, 'learning_rate': 7.744170747470685e-07, 'epoch': 0.150257662055872, 'num_input_tokens_seen': 2839543808, 'completed': '36.72% (1_354 / 3_687)', 'remaining time': '20:36:46', 'throughput': '8508.38', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 20:21:59,571 >> {'loss': 0.954, 'grad_norm': 23.312227249145508, 'learning_rate': 7.740670787087972e-07, 'epoch': 0.15052888527257932, 'num_input_tokens_seen': 2841640960, 'completed': '36.75% (1_355 / 3_687)', 'remaining time': '20:35:48', 'throughput': '10225.02', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 20:22:31,015 >> {'loss': 0.3264, 'grad_norm': 13.209382057189941, 'learning_rate': 7.737169023498672e-07, 'epoch': 0.15080010848928668, 'num_input_tokens_seen': 2843738112, 'completed': '36.78% (1_356 / 3_687)', 'remaining time': '20:35:15', 'throughput': '8336.94', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 20:23:01,263 >> {'loss': 0.3447, 'grad_norm': 15.09355354309082, 'learning_rate': 7.733665459520873e-07, 'epoch': 0.15107133170599404, 'num_input_tokens_seen': 2845835264, 'completed': '36.80% (1_357 / 3_687)', 'remaining time': '20:34:37', 'throughput': '8666.37', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 20:23:32,135 >> {'loss': 0.3267, 'grad_norm': 12.502537727355957, 'learning_rate': 7.730160097974109e-07, 'epoch': 0.15134255492270138, 'num_input_tokens_seen': 2847932416, 'completed': '36.83% (1_358 / 3_687)', 'remaining time': '20:34:01', 'throughput': '8491.43', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 20:24:03,818 >> {'loss': 0.4452, 'grad_norm': 17.678531646728516, 'learning_rate': 7.726652941679365e-07, 'epoch': 0.15161377813940874, 'num_input_tokens_seen': 2850029568, 'completed': '36.86% (1_359 / 3_687)', 'remaining time': '20:33:29', 'throughput': '8273.99', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 20:24:34,855 >> {'loss': 0.443, 'grad_norm': 14.068532943725586, 'learning_rate': 7.723143993459061e-07, 'epoch': 0.15188500135611607, 'num_input_tokens_seen': 2852126720, 'completed': '36.89% (1_360 / 3_687)', 'remaining time': '20:32:54', 'throughput': '8446.01', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 20:25:04,826 >> {'loss': 0.4935, 'grad_norm': 17.834699630737305, 'learning_rate': 7.719633256137067e-07, 'epoch': 0.15215622457282343, 'num_input_tokens_seen': 2854223872, 'completed': '36.91% (1_361 / 3_687)', 'remaining time': '20:32:15', 'throughput': '8746.62', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 20:25:35,913 >> {'loss': 0.2926, 'grad_norm': 23.74162483215332, 'learning_rate': 7.716120732538696e-07, 'epoch': 0.1524274477895308, 'num_input_tokens_seen': 2856321024, 'completed': '36.94% (1_362 / 3_687)', 'remaining time': '20:31:40', 'throughput': '8432.58', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 20:26:06,658 >> {'loss': 0.6429, 'grad_norm': 21.309246063232422, 'learning_rate': 7.712606425490687e-07, 'epoch': 0.15269867100623813, 'num_input_tokens_seen': 2858418176, 'completed': '36.97% (1_363 / 3_687)', 'remaining time': '20:31:04', 'throughput': '8526.46', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 20:26:35,044 >> {'loss': 0.522, 'grad_norm': 28.85659408569336, 'learning_rate': 7.70909033782122e-07, 'epoch': 0.1529698942229455, 'num_input_tokens_seen': 2860515328, 'completed': '36.99% (1_364 / 3_687)', 'remaining time': '20:30:18', 'throughput': '9235.01', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 20:27:05,684 >> {'loss': 0.5151, 'grad_norm': 16.417978286743164, 'learning_rate': 7.705572472359913e-07, 'epoch': 0.15324111743965282, 'num_input_tokens_seen': 2862612480, 'completed': '37.02% (1_365 / 3_687)', 'remaining time': '20:29:42', 'throughput': '8555.50', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 20:27:35,262 >> {'loss': 0.4831, 'grad_norm': 15.138096809387207, 'learning_rate': 7.702052831937811e-07, 'epoch': 0.15351234065636019, 'num_input_tokens_seen': 2864709632, 'completed': '37.05% (1_366 / 3_687)', 'remaining time': '20:29:01', 'throughput': '8862.83', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 20:28:07,030 >> {'loss': 0.6095, 'grad_norm': 17.662302017211914, 'learning_rate': 7.698531419387382e-07, 'epoch': 0.15378356387306755, 'num_input_tokens_seen': 2866806784, 'completed': '37.08% (1_367 / 3_687)', 'remaining time': '20:28:29', 'throughput': '8251.94', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 20:28:38,893 >> {'loss': 0.3174, 'grad_norm': 13.90149211883545, 'learning_rate': 7.695008237542526e-07, 'epoch': 0.15405478708977488, 'num_input_tokens_seen': 2868903936, 'completed': '37.10% (1_368 / 3_687)', 'remaining time': '20:27:58', 'throughput': '8227.12', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 20:29:08,433 >> {'loss': 0.9373, 'grad_norm': 25.922029495239258, 'learning_rate': 7.691483289238569e-07, 'epoch': 0.15432601030648224, 'num_input_tokens_seen': 2871001088, 'completed': '37.13% (1_369 / 3_687)', 'remaining time': '20:27:17', 'throughput': '8874.11', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 20:29:39,788 >> {'loss': 0.5871, 'grad_norm': 18.99474334716797, 'learning_rate': 7.687956577312251e-07, 'epoch': 0.15459723352318958, 'num_input_tokens_seen': 2873098240, 'completed': '37.16% (1_370 / 3_687)', 'remaining time': '20:26:43', 'throughput': '8360.56', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 20:30:11,591 >> {'loss': 0.3569, 'grad_norm': 16.25048828125, 'learning_rate': 7.684428104601739e-07, 'epoch': 0.15486845673989694, 'num_input_tokens_seen': 2875195392, 'completed': '37.18% (1_371 / 3_687)', 'remaining time': '20:26:12', 'throughput': '8242.93', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 20:30:43,685 >> {'loss': 0.3744, 'grad_norm': 12.14373779296875, 'learning_rate': 7.680897873946611e-07, 'epoch': 0.1551396799566043, 'num_input_tokens_seen': 2877292544, 'completed': '37.21% (1_372 / 3_687)', 'remaining time': '20:25:41', 'throughput': '8167.75', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 20:31:17,336 >> {'loss': 0.4764, 'grad_norm': 15.390382766723633, 'learning_rate': 7.677365888187864e-07, 'epoch': 0.15541090317331163, 'num_input_tokens_seen': 2879389696, 'completed': '37.24% (1_373 / 3_687)', 'remaining time': '20:25:17', 'throughput': '7790.15', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 20:31:48,396 >> {'loss': 0.4289, 'grad_norm': 16.843217849731445, 'learning_rate': 7.673832150167906e-07, 'epoch': 0.155682126390019, 'num_input_tokens_seen': 2881486848, 'completed': '37.27% (1_374 / 3_687)', 'remaining time': '20:24:43', 'throughput': '8439.99', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 20:32:18,937 >> {'loss': 0.3706, 'grad_norm': 13.040255546569824, 'learning_rate': 7.670296662730552e-07, 'epoch': 0.15595334960672633, 'num_input_tokens_seen': 2883584000, 'completed': '37.29% (1_375 / 3_687)', 'remaining time': '20:24:06', 'throughput': '8583.21', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 20:32:47,997 >> {'loss': 0.5491, 'grad_norm': 16.71036148071289, 'learning_rate': 7.66675942872103e-07, 'epoch': 0.1562245728234337, 'num_input_tokens_seen': 2885681152, 'completed': '37.32% (1_376 / 3_687)', 'remaining time': '20:23:23', 'throughput': '9021.07', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 20:33:21,390 >> {'loss': 0.401, 'grad_norm': 14.318278312683105, 'learning_rate': 7.663220450985973e-07, 'epoch': 0.15649579604014105, 'num_input_tokens_seen': 2887778304, 'completed': '37.35% (1_377 / 3_687)', 'remaining time': '20:22:58', 'throughput': '7850.19', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 20:33:51,977 >> {'loss': 0.6904, 'grad_norm': 20.72394371032715, 'learning_rate': 7.659679732373413e-07, 'epoch': 0.15676701925684838, 'num_input_tokens_seen': 2889875456, 'completed': '37.37% (1_378 / 3_687)', 'remaining time': '20:22:22', 'throughput': '8570.28', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 20:34:22,699 >> {'loss': 0.3329, 'grad_norm': 12.829943656921387, 'learning_rate': 7.656137275732786e-07, 'epoch': 0.15703824247355574, 'num_input_tokens_seen': 2891972608, 'completed': '37.40% (1_379 / 3_687)', 'remaining time': '20:21:46', 'throughput': '8532.99', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 20:34:50,863 >> {'loss': 0.6388, 'grad_norm': 19.38332748413086, 'learning_rate': 7.652593083914927e-07, 'epoch': 0.15730946569026308, 'num_input_tokens_seen': 2894069760, 'completed': '37.43% (1_380 / 3_687)', 'remaining time': '20:21:00', 'throughput': '9307.56', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 20:35:20,022 >> {'loss': 0.5891, 'grad_norm': 19.630373001098633, 'learning_rate': 7.649047159772064e-07, 'epoch': 0.15758068890697044, 'num_input_tokens_seen': 2896166912, 'completed': '37.46% (1_381 / 3_687)', 'remaining time': '20:20:18', 'throughput': '8990.21', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 20:35:50,986 >> {'loss': 0.42, 'grad_norm': 15.700737953186035, 'learning_rate': 7.645499506157827e-07, 'epoch': 0.15785191212367777, 'num_input_tokens_seen': 2898264064, 'completed': '37.48% (1_382 / 3_687)', 'remaining time': '20:19:43', 'throughput': '8466.16', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 20:36:22,796 >> {'loss': 0.5836, 'grad_norm': 17.65544319152832, 'learning_rate': 7.641950125927228e-07, 'epoch': 0.15812313534038513, 'num_input_tokens_seen': 2900361216, 'completed': '37.51% (1_383 / 3_687)', 'remaining time': '20:19:11', 'throughput': '8240.99', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 20:36:52,381 >> {'loss': 0.3025, 'grad_norm': 12.071609497070312, 'learning_rate': 7.638399021936675e-07, 'epoch': 0.1583943585570925, 'num_input_tokens_seen': 2902458368, 'completed': '37.54% (1_384 / 3_687)', 'remaining time': '20:18:31', 'throughput': '8860.65', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 20:37:24,077 >> {'loss': 0.4026, 'grad_norm': 16.76353645324707, 'learning_rate': 7.634846197043963e-07, 'epoch': 0.15866558177379983, 'num_input_tokens_seen': 2904555520, 'completed': '37.56% (1_385 / 3_687)', 'remaining time': '20:17:59', 'throughput': '8270.45', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 20:37:54,314 >> {'loss': 0.642, 'grad_norm': 18.88435173034668, 'learning_rate': 7.631291654108266e-07, 'epoch': 0.1589368049905072, 'num_input_tokens_seen': 2906652672, 'completed': '37.59% (1_386 / 3_687)', 'remaining time': '20:17:21', 'throughput': '8669.80', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 20:38:22,988 >> {'loss': 0.8431, 'grad_norm': 25.31294822692871, 'learning_rate': 7.627735395990149e-07, 'epoch': 0.15920802820721452, 'num_input_tokens_seen': 2908749824, 'completed': '37.62% (1_387 / 3_687)', 'remaining time': '20:16:37', 'throughput': '9142.09', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 20:38:49,758 >> {'loss': 0.9865, 'grad_norm': 30.660036087036133, 'learning_rate': 7.624177425551552e-07, 'epoch': 0.1594792514239219, 'num_input_tokens_seen': 2910846976, 'completed': '37.65% (1_388 / 3_687)', 'remaining time': '20:15:46', 'throughput': '9792.47', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 20:39:20,164 >> {'loss': 0.7311, 'grad_norm': 23.799144744873047, 'learning_rate': 7.620617745655793e-07, 'epoch': 0.15975047464062925, 'num_input_tokens_seen': 2912944128, 'completed': '37.67% (1_389 / 3_687)', 'remaining time': '20:15:09', 'throughput': '8621.59', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 20:39:56,028 >> {'loss': 0.4711, 'grad_norm': 21.239885330200195, 'learning_rate': 7.617056359167568e-07, 'epoch': 0.16002169785733658, 'num_input_tokens_seen': 2915041280, 'completed': '37.70% (1_390 / 3_687)', 'remaining time': '20:14:54', 'throughput': '7309.29', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 20:40:24,112 >> {'loss': 0.5114, 'grad_norm': 19.135162353515625, 'learning_rate': 7.613493268952947e-07, 'epoch': 0.16029292107404394, 'num_input_tokens_seen': 2917138432, 'completed': '37.73% (1_391 / 3_687)', 'remaining time': '20:14:08', 'throughput': '9334.21', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 20:40:55,803 >> {'loss': 0.3154, 'grad_norm': 16.555891036987305, 'learning_rate': 7.609928477879365e-07, 'epoch': 0.16056414429075128, 'num_input_tokens_seen': 2919235584, 'completed': '37.75% (1_392 / 3_687)', 'remaining time': '20:13:36', 'throughput': '8271.93', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 20:41:26,096 >> {'loss': 0.5111, 'grad_norm': 28.75040626525879, 'learning_rate': 7.606361988815633e-07, 'epoch': 0.16083536750745864, 'num_input_tokens_seen': 2921332736, 'completed': '37.78% (1_393 / 3_687)', 'remaining time': '20:12:59', 'throughput': '8653.80', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 20:41:58,125 >> {'loss': 0.2485, 'grad_norm': 22.3262939453125, 'learning_rate': 7.602793804631927e-07, 'epoch': 0.161106590724166, 'num_input_tokens_seen': 2923429888, 'completed': '37.81% (1_394 / 3_687)', 'remaining time': '20:12:28', 'throughput': '8184.57', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 20:42:29,054 >> {'loss': 0.4578, 'grad_norm': 15.991744041442871, 'learning_rate': 7.599223928199781e-07, 'epoch': 0.16137781394087333, 'num_input_tokens_seen': 2925527040, 'completed': '37.84% (1_395 / 3_687)', 'remaining time': '20:11:53', 'throughput': '8475.68', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 20:42:55,971 >> {'loss': 0.5837, 'grad_norm': 19.883453369140625, 'learning_rate': 7.595652362392103e-07, 'epoch': 0.1616490371575807, 'num_input_tokens_seen': 2927624192, 'completed': '37.86% (1_396 / 3_687)', 'remaining time': '20:11:03', 'throughput': '9738.99', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 20:43:24,520 >> {'loss': 0.6984, 'grad_norm': 18.910655975341797, 'learning_rate': 7.592079110083146e-07, 'epoch': 0.16192026037428803, 'num_input_tokens_seen': 2929721344, 'completed': '37.89% (1_397 / 3_687)', 'remaining time': '20:10:19', 'throughput': '9182.04', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 20:43:55,413 >> {'loss': 0.4046, 'grad_norm': 14.154722213745117, 'learning_rate': 7.588504174148532e-07, 'epoch': 0.1621914835909954, 'num_input_tokens_seen': 2931818496, 'completed': '37.92% (1_398 / 3_687)', 'remaining time': '20:09:44', 'throughput': '8485.57', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 20:44:23,478 >> {'loss': 0.729, 'grad_norm': 24.498065948486328, 'learning_rate': 7.584927557465233e-07, 'epoch': 0.16246270680770275, 'num_input_tokens_seen': 2933915648, 'completed': '37.94% (1_399 / 3_687)', 'remaining time': '20:08:59', 'throughput': '9340.58', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 20:44:54,237 >> {'loss': 0.3138, 'grad_norm': 18.18712043762207, 'learning_rate': 7.581349262911573e-07, 'epoch': 0.16273393002441008, 'num_input_tokens_seen': 2936012800, 'completed': '37.97% (1_400 / 3_687)', 'remaining time': '20:08:24', 'throughput': '8522.50', 'gpu_mem_free': '5581MB'} +/scratch3/workspace/ctpham_umass_edu-ft/envs/prolong-final/lib/python3.10/site-packages/torch/distributed/fsdp/fully_sharded_data_parallel.py:689: FutureWarning: FSDP.state_dict_type() and FSDP.set_state_dict_type() are being deprecated. Please use APIs, get_state_dict() and set_state_dict(), which can support different parallelisms, FSDP1, FSDP2, DDP. API doc: https://pytorch.org/docs/stable/distributed.checkpoint.html#torch.distributed.checkpoint.state_dict.get_state_dict .Tutorial: https://pytorch.org/tutorials/recipes/distributed_checkpoint_recipe.html . + warnings.warn( +[INFO|trainer.py:3503] 2024-12-25 20:45:27,790 >> Saving model checkpoint to /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/checkpoint-1400 +[INFO|configuration_utils.py:472] 2024-12-25 20:45:27,794 >> Configuration saved in /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/checkpoint-1400/config.json +[INFO|configuration_utils.py:807] 2024-12-25 20:45:27,795 >> Configuration saved in /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/checkpoint-1400/generation_config.json +[INFO|modeling_utils.py:2807] 2024-12-25 20:46:38,615 >> The model is bigger than the maximum size per checkpoint (5GB) and is going to be split in 7 checkpoint shards. You can find where each parameters has been saved in the index located at /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/checkpoint-1400/model.safetensors.index.json. +[INFO|tokenization_utils_base.py:2684] 2024-12-25 20:46:38,619 >> tokenizer config file saved in /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/checkpoint-1400/tokenizer_config.json +[INFO|tokenization_utils_base.py:2693] 2024-12-25 20:46:38,620 >> Special tokens file saved in /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/checkpoint-1400/special_tokens_map.json +/scratch3/workspace/ctpham_umass_edu-ft/envs/prolong-final/lib/python3.10/site-packages/torch/distributed/fsdp/fully_sharded_data_parallel.py:689: FutureWarning: FSDP.state_dict_type() and FSDP.set_state_dict_type() are being deprecated. Please use APIs, get_state_dict() and set_state_dict(), which can support different parallelisms, FSDP1, FSDP2, DDP. API doc: https://pytorch.org/docs/stable/distributed.checkpoint.html#torch.distributed.checkpoint.state_dict.get_state_dict .Tutorial: https://pytorch.org/tutorials/recipes/distributed_checkpoint_recipe.html . + warnings.warn( +[WARNING|trainer.py:868] 2024-12-25 20:50:21,744 >> Save streaming dataset state: {'epoch': 0, 'sample_in_epoch': 2800, 'num_canonical_nodes': 1, 'shuffle_seed': 42, 'initial_physical_nodes': 1} +/scratch3/workspace/ctpham_umass_edu-ft/envs/prolong-final/lib/python3.10/site-packages/torch/utils/checkpoint.py:1399: FutureWarning: `torch.cpu.amp.autocast(args...)` is deprecated. Please use `torch.amp.autocast('cpu', args...)` instead. + with device_autocast_ctx, torch.cpu.amp.autocast(**cpu_autocast_kwargs), recompute_context: # type: ignore[attr-defined] +[INFO|trainer.py:175] 2024-12-25 20:50:52,140 >> {'loss': 0.4787, 'grad_norm': 16.402795791625977, 'learning_rate': 7.577769293367226e-07, 'epoch': 0.16300515324111745, 'num_input_tokens_seen': 2938109952, 'completed': '38.00% (1_401 / 3_687)', 'remaining time': '20:28:33', 'throughput': '732.44', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 20:51:25,997 >> {'loss': 0.4823, 'grad_norm': 16.0742130279541, 'learning_rate': 7.574187651713218e-07, 'epoch': 0.16327637645782478, 'num_input_tokens_seen': 2940207104, 'completed': '38.03% (1_402 / 3_687)', 'remaining time': '20:28:06', 'throughput': '7742.78', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 20:51:56,166 >> {'loss': 0.392, 'grad_norm': 16.962581634521484, 'learning_rate': 7.570604340831916e-07, 'epoch': 0.16354759967453214, 'num_input_tokens_seen': 2942304256, 'completed': '38.05% (1_403 / 3_687)', 'remaining time': '20:27:26', 'throughput': '8689.10', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 20:52:27,735 >> {'loss': 0.262, 'grad_norm': 12.302724838256836, 'learning_rate': 7.567019363607032e-07, 'epoch': 0.1638188228912395, 'num_input_tokens_seen': 2944401408, 'completed': '38.08% (1_404 / 3_687)', 'remaining time': '20:26:52', 'throughput': '8303.98', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 20:52:59,842 >> {'loss': 0.3794, 'grad_norm': 13.02363109588623, 'learning_rate': 7.563432722923621e-07, 'epoch': 0.16409004610794684, 'num_input_tokens_seen': 2946498560, 'completed': '38.11% (1_405 / 3_687)', 'remaining time': '20:26:19', 'throughput': '8164.67', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 20:53:31,447 >> {'loss': 0.2997, 'grad_norm': 13.745916366577148, 'learning_rate': 7.559844421668074e-07, 'epoch': 0.1643612693246542, 'num_input_tokens_seen': 2948595712, 'completed': '38.13% (1_406 / 3_687)', 'remaining time': '20:25:44', 'throughput': '8294.27', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 20:53:59,596 >> {'loss': 0.5652, 'grad_norm': 16.339326858520508, 'learning_rate': 7.556254462728122e-07, 'epoch': 0.16463249254136153, 'num_input_tokens_seen': 2950692864, 'completed': '38.16% (1_407 / 3_687)', 'remaining time': '20:24:56', 'throughput': '9312.87', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 20:54:30,469 >> {'loss': 0.4496, 'grad_norm': 16.774593353271484, 'learning_rate': 7.552662848992822e-07, 'epoch': 0.1649037157580689, 'num_input_tokens_seen': 2952790016, 'completed': '38.19% (1_408 / 3_687)', 'remaining time': '20:24:19', 'throughput': '8490.87', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 20:55:01,429 >> {'loss': 0.4142, 'grad_norm': 14.789047241210938, 'learning_rate': 7.54906958335257e-07, 'epoch': 0.16517493897477625, 'num_input_tokens_seen': 2954887168, 'completed': '38.22% (1_409 / 3_687)', 'remaining time': '20:23:42', 'throughput': '8467.29', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 20:55:31,575 >> {'loss': 0.3419, 'grad_norm': 15.732011795043945, 'learning_rate': 7.545474668699091e-07, 'epoch': 0.1654461621914836, 'num_input_tokens_seen': 2956984320, 'completed': '38.24% (1_410 / 3_687)', 'remaining time': '20:23:02', 'throughput': '8695.80', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 20:56:01,476 >> {'loss': 0.3786, 'grad_norm': 14.704493522644043, 'learning_rate': 7.541878107925435e-07, 'epoch': 0.16571738540819095, 'num_input_tokens_seen': 2959081472, 'completed': '38.27% (1_411 / 3_687)', 'remaining time': '20:22:21', 'throughput': '8767.00', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 20:56:32,158 >> {'loss': 0.2512, 'grad_norm': 13.177338600158691, 'learning_rate': 7.538279903925977e-07, 'epoch': 0.16598860862489828, 'num_input_tokens_seen': 2961178624, 'completed': '38.30% (1_412 / 3_687)', 'remaining time': '20:21:43', 'throughput': '8544.09', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 20:56:59,335 >> {'loss': 0.4833, 'grad_norm': 17.341163635253906, 'learning_rate': 7.534680059596414e-07, 'epoch': 0.16625983184160564, 'num_input_tokens_seen': 2963275776, 'completed': '38.32% (1_413 / 3_687)', 'remaining time': '20:20:52', 'throughput': '9645.58', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 20:57:29,592 >> {'loss': 0.6702, 'grad_norm': 23.333568572998047, 'learning_rate': 7.531078577833765e-07, 'epoch': 0.166531055058313, 'num_input_tokens_seen': 2965372928, 'completed': '38.35% (1_414 / 3_687)', 'remaining time': '20:20:13', 'throughput': '8663.95', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 20:58:00,737 >> {'loss': 0.3171, 'grad_norm': 12.770888328552246, 'learning_rate': 7.527475461536363e-07, 'epoch': 0.16680227827502034, 'num_input_tokens_seen': 2967470080, 'completed': '38.38% (1_415 / 3_687)', 'remaining time': '20:19:37', 'throughput': '8416.77', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 20:58:34,854 >> {'loss': 0.3315, 'grad_norm': 16.709857940673828, 'learning_rate': 7.523870713603864e-07, 'epoch': 0.1670735014917277, 'num_input_tokens_seen': 2969567232, 'completed': '38.41% (1_416 / 3_687)', 'remaining time': '20:19:12', 'throughput': '7683.68', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 20:59:05,088 >> {'loss': 0.3777, 'grad_norm': 14.015904426574707, 'learning_rate': 7.520264336937227e-07, 'epoch': 0.16734472470843503, 'num_input_tokens_seen': 2971664384, 'completed': '38.43% (1_417 / 3_687)', 'remaining time': '20:18:32', 'throughput': '8670.71', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 20:59:36,593 >> {'loss': 0.3262, 'grad_norm': 12.915022850036621, 'learning_rate': 7.516656334438727e-07, 'epoch': 0.1676159479251424, 'num_input_tokens_seen': 2973761536, 'completed': '38.46% (1_418 / 3_687)', 'remaining time': '20:17:57', 'throughput': '8320.57', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 21:00:07,850 >> {'loss': 0.5589, 'grad_norm': 19.277992248535156, 'learning_rate': 7.513046709011951e-07, 'epoch': 0.16788717114184976, 'num_input_tokens_seen': 2975858688, 'completed': '38.49% (1_419 / 3_687)', 'remaining time': '20:17:22', 'throughput': '8386.83', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 21:00:38,844 >> {'loss': 0.3394, 'grad_norm': 15.75462532043457, 'learning_rate': 7.509435463561785e-07, 'epoch': 0.1681583943585571, 'num_input_tokens_seen': 2977955840, 'completed': '38.51% (1_420 / 3_687)', 'remaining time': '20:16:45', 'throughput': '8457.78', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 21:01:09,050 >> {'loss': 0.7979, 'grad_norm': 21.828617095947266, 'learning_rate': 7.505822600994423e-07, 'epoch': 0.16842961757526445, 'num_input_tokens_seen': 2980052992, 'completed': '38.54% (1_421 / 3_687)', 'remaining time': '20:16:05', 'throughput': '8678.75', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 21:01:40,278 >> {'loss': 0.4936, 'grad_norm': 24.69605255126953, 'learning_rate': 7.502208124217357e-07, 'epoch': 0.16870084079197178, 'num_input_tokens_seen': 2982150144, 'completed': '38.57% (1_422 / 3_687)', 'remaining time': '20:15:30', 'throughput': '8394.29', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 21:02:08,418 >> {'loss': 0.6869, 'grad_norm': 21.24989891052246, 'learning_rate': 7.498592036139383e-07, 'epoch': 0.16897206400867915, 'num_input_tokens_seen': 2984247296, 'completed': '38.60% (1_423 / 3_687)', 'remaining time': '20:14:43', 'throughput': '9315.95', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 21:02:38,938 >> {'loss': 0.3728, 'grad_norm': 18.29302215576172, 'learning_rate': 7.494974339670591e-07, 'epoch': 0.16924328722538648, 'num_input_tokens_seen': 2986344448, 'completed': '38.62% (1_424 / 3_687)', 'remaining time': '20:14:05', 'throughput': '8589.29', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 21:03:08,246 >> {'loss': 0.3315, 'grad_norm': 15.673701286315918, 'learning_rate': 7.491355037722365e-07, 'epoch': 0.16951451044209384, 'num_input_tokens_seen': 2988441600, 'completed': '38.65% (1_425 / 3_687)', 'remaining time': '20:13:22', 'throughput': '8944.22', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 21:03:38,551 >> {'loss': 0.5771, 'grad_norm': 20.936105728149414, 'learning_rate': 7.487734133207382e-07, 'epoch': 0.1697857336588012, 'num_input_tokens_seen': 2990538752, 'completed': '38.68% (1_426 / 3_687)', 'remaining time': '20:12:43', 'throughput': '8650.22', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 21:04:07,986 >> {'loss': 0.2192, 'grad_norm': 10.056077003479004, 'learning_rate': 7.484111629039607e-07, 'epoch': 0.17005695687550854, 'num_input_tokens_seen': 2992635904, 'completed': '38.70% (1_427 / 3_687)', 'remaining time': '20:12:01', 'throughput': '8905.77', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 21:04:37,237 >> {'loss': 0.7378, 'grad_norm': 21.529075622558594, 'learning_rate': 7.480487528134292e-07, 'epoch': 0.1703281800922159, 'num_input_tokens_seen': 2994733056, 'completed': '38.73% (1_428 / 3_687)', 'remaining time': '20:11:18', 'throughput': '8962.10', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 21:05:09,316 >> {'loss': 0.3931, 'grad_norm': 16.297054290771484, 'learning_rate': 7.47686183340798e-07, 'epoch': 0.17059940330892323, 'num_input_tokens_seen': 2996830208, 'completed': '38.76% (1_429 / 3_687)', 'remaining time': '20:10:46', 'throughput': '8171.90', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 21:05:42,893 >> {'loss': 0.6421, 'grad_norm': 20.00302505493164, 'learning_rate': 7.473234547778489e-07, 'epoch': 0.1708706265256306, 'num_input_tokens_seen': 2998927360, 'completed': '38.78% (1_430 / 3_687)', 'remaining time': '20:10:19', 'throughput': '7807.22', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 21:06:14,005 >> {'loss': 0.4505, 'grad_norm': 16.071239471435547, 'learning_rate': 7.46960567416492e-07, 'epoch': 0.17114184974233795, 'num_input_tokens_seen': 3001024512, 'completed': '38.81% (1_431 / 3_687)', 'remaining time': '20:09:43', 'throughput': '8425.69', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 21:06:43,639 >> {'loss': 0.2977, 'grad_norm': 14.117545127868652, 'learning_rate': 7.465975215487655e-07, 'epoch': 0.1714130729590453, 'num_input_tokens_seen': 3003121664, 'completed': '38.84% (1_432 / 3_687)', 'remaining time': '20:09:01', 'throughput': '8845.99', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 21:07:13,192 >> {'loss': 0.6603, 'grad_norm': 21.76426124572754, 'learning_rate': 7.462343174668346e-07, 'epoch': 0.17168429617575265, 'num_input_tokens_seen': 3005218816, 'completed': '38.87% (1_433 / 3_687)', 'remaining time': '20:08:20', 'throughput': '8870.41', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 21:07:42,162 >> {'loss': 0.5771, 'grad_norm': 20.707962036132812, 'learning_rate': 7.458709554629924e-07, 'epoch': 0.17195551939245998, 'num_input_tokens_seen': 3007315968, 'completed': '38.89% (1_434 / 3_687)', 'remaining time': '20:07:36', 'throughput': '9048.89', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 21:08:11,132 >> {'loss': 0.6713, 'grad_norm': 19.084461212158203, 'learning_rate': 7.455074358296586e-07, 'epoch': 0.17222674260916734, 'num_input_tokens_seen': 3009413120, 'completed': '38.92% (1_435 / 3_687)', 'remaining time': '20:06:53', 'throughput': '9048.75', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 21:08:44,950 >> {'loss': 0.3933, 'grad_norm': 13.040257453918457, 'learning_rate': 7.451437588593802e-07, 'epoch': 0.1724979658258747, 'num_input_tokens_seen': 3011510272, 'completed': '38.95% (1_436 / 3_687)', 'remaining time': '20:06:27', 'throughput': '7751.78', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 21:09:15,209 >> {'loss': 0.5142, 'grad_norm': 16.921184539794922, 'learning_rate': 7.447799248448303e-07, 'epoch': 0.17276918904258204, 'num_input_tokens_seen': 3013607424, 'completed': '38.97% (1_437 / 3_687)', 'remaining time': '20:05:48', 'throughput': '8663.22', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 21:09:49,155 >> {'loss': 0.5446, 'grad_norm': 16.22494125366211, 'learning_rate': 7.444159340788088e-07, 'epoch': 0.1730404122592894, 'num_input_tokens_seen': 3015704576, 'completed': '39.00% (1_438 / 3_687)', 'remaining time': '20:05:22', 'throughput': '7722.36', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 21:10:16,513 >> {'loss': 0.3896, 'grad_norm': 13.45175838470459, 'learning_rate': 7.440517868542417e-07, 'epoch': 0.17331163547599673, 'num_input_tokens_seen': 3017801728, 'completed': '39.03% (1_439 / 3_687)', 'remaining time': '20:04:33', 'throughput': '9581.98', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 21:10:48,579 >> {'loss': 0.3761, 'grad_norm': 14.45242691040039, 'learning_rate': 7.436874834641807e-07, 'epoch': 0.1735828586927041, 'num_input_tokens_seen': 3019898880, 'completed': '39.06% (1_440 / 3_687)', 'remaining time': '20:04:00', 'throughput': '8174.96', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 21:11:20,167 >> {'loss': 0.5964, 'grad_norm': 21.77591323852539, 'learning_rate': 7.433230242018035e-07, 'epoch': 0.17385408190941146, 'num_input_tokens_seen': 3021996032, 'completed': '39.08% (1_441 / 3_687)', 'remaining time': '20:03:26', 'throughput': '8298.94', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 21:11:53,033 >> {'loss': 0.3924, 'grad_norm': 16.96351432800293, 'learning_rate': 7.429584093604128e-07, 'epoch': 0.1741253051261188, 'num_input_tokens_seen': 3024093184, 'completed': '39.11% (1_442 / 3_687)', 'remaining time': '20:02:57', 'throughput': '7976.25', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 21:12:22,648 >> {'loss': 0.6515, 'grad_norm': 19.076356887817383, 'learning_rate': 7.425936392334368e-07, 'epoch': 0.17439652834282615, 'num_input_tokens_seen': 3026190336, 'completed': '39.14% (1_443 / 3_687)', 'remaining time': '20:02:16', 'throughput': '8851.59', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 21:12:50,334 >> {'loss': 0.6622, 'grad_norm': 18.412277221679688, 'learning_rate': 7.422287141144287e-07, 'epoch': 0.17466775155953349, 'num_input_tokens_seen': 3028287488, 'completed': '39.16% (1_444 / 3_687)', 'remaining time': '20:01:28', 'throughput': '9468.61', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 21:13:17,575 >> {'loss': 0.7812, 'grad_norm': 19.85443687438965, 'learning_rate': 7.418636342970665e-07, 'epoch': 0.17493897477624085, 'num_input_tokens_seen': 3030384640, 'completed': '39.19% (1_445 / 3_687)', 'remaining time': '20:00:39', 'throughput': '9623.07', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 21:13:47,353 >> {'loss': 0.3791, 'grad_norm': 18.57454490661621, 'learning_rate': 7.414984000751521e-07, 'epoch': 0.1752101979929482, 'num_input_tokens_seen': 3032481792, 'completed': '39.22% (1_446 / 3_687)', 'remaining time': '19:59:58', 'throughput': '8803.32', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 21:14:17,011 >> {'loss': 0.3755, 'grad_norm': 13.877761840820312, 'learning_rate': 7.411330117426125e-07, 'epoch': 0.17548142120965554, 'num_input_tokens_seen': 3034578944, 'completed': '39.25% (1_447 / 3_687)', 'remaining time': '19:59:18', 'throughput': '8838.82', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 21:14:46,993 >> {'loss': 0.2692, 'grad_norm': 11.10472583770752, 'learning_rate': 7.407674695934983e-07, 'epoch': 0.1757526444263629, 'num_input_tokens_seen': 3036676096, 'completed': '39.27% (1_448 / 3_687)', 'remaining time': '19:58:38', 'throughput': '8743.52', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 21:15:19,343 >> {'loss': 0.4897, 'grad_norm': 16.257774353027344, 'learning_rate': 7.404017739219836e-07, 'epoch': 0.17602386764307024, 'num_input_tokens_seen': 3038773248, 'completed': '39.30% (1_449 / 3_687)', 'remaining time': '19:58:07', 'throughput': '8103.32', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 21:15:51,049 >> {'loss': 0.2769, 'grad_norm': 13.585163116455078, 'learning_rate': 7.400359250223667e-07, 'epoch': 0.1762950908597776, 'num_input_tokens_seen': 3040870400, 'completed': '39.33% (1_450 / 3_687)', 'remaining time': '19:57:33', 'throughput': '8267.87', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 21:16:20,439 >> {'loss': 0.5257, 'grad_norm': 16.29515266418457, 'learning_rate': 7.396699231890689e-07, 'epoch': 0.17656631407648496, 'num_input_tokens_seen': 3042967552, 'completed': '39.35% (1_451 / 3_687)', 'remaining time': '19:56:52', 'throughput': '8919.47', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 21:16:49,023 >> {'loss': 0.5871, 'grad_norm': 19.343111038208008, 'learning_rate': 7.393037687166342e-07, 'epoch': 0.1768375372931923, 'num_input_tokens_seen': 3045064704, 'completed': '39.38% (1_452 / 3_687)', 'remaining time': '19:56:08', 'throughput': '9171.19', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 21:17:23,844 >> {'loss': 0.5044, 'grad_norm': 17.655189514160156, 'learning_rate': 7.3893746189973e-07, 'epoch': 0.17710876050989965, 'num_input_tokens_seen': 3047161856, 'completed': '39.41% (1_453 / 3_687)', 'remaining time': '19:55:45', 'throughput': '7528.16', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 21:17:52,835 >> {'loss': 0.4438, 'grad_norm': 15.424549102783203, 'learning_rate': 7.385710030331461e-07, 'epoch': 0.177379983726607, 'num_input_tokens_seen': 3049259008, 'completed': '39.44% (1_454 / 3_687)', 'remaining time': '19:55:02', 'throughput': '9042.53', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 21:18:22,224 >> {'loss': 0.3884, 'grad_norm': 15.889403343200684, 'learning_rate': 7.382043924117945e-07, 'epoch': 0.17765120694331435, 'num_input_tokens_seen': 3051356160, 'completed': '39.46% (1_455 / 3_687)', 'remaining time': '19:54:21', 'throughput': '8919.53', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 21:18:52,081 >> {'loss': 0.2612, 'grad_norm': 11.11455249786377, 'learning_rate': 7.378376303307099e-07, 'epoch': 0.1779224301600217, 'num_input_tokens_seen': 3053453312, 'completed': '39.49% (1_456 / 3_687)', 'remaining time': '19:53:41', 'throughput': '8780.20', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 21:19:24,449 >> {'loss': 0.45, 'grad_norm': 18.22006607055664, 'learning_rate': 7.374707170850479e-07, 'epoch': 0.17819365337672904, 'num_input_tokens_seen': 3055550464, 'completed': '39.52% (1_457 / 3_687)', 'remaining time': '19:53:10', 'throughput': '8098.83', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 21:19:54,924 >> {'loss': 0.4225, 'grad_norm': 14.16891098022461, 'learning_rate': 7.371036529700866e-07, 'epoch': 0.1784648765934364, 'num_input_tokens_seen': 3057647616, 'completed': '39.54% (1_458 / 3_687)', 'remaining time': '19:52:32', 'throughput': '8601.96', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 21:20:23,997 >> {'loss': 0.3465, 'grad_norm': 13.474692344665527, 'learning_rate': 7.367364382812253e-07, 'epoch': 0.17873609981014374, 'num_input_tokens_seen': 3059744768, 'completed': '39.57% (1_459 / 3_687)', 'remaining time': '19:51:50', 'throughput': '9016.64', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 21:20:54,636 >> {'loss': 0.6558, 'grad_norm': 19.395767211914062, 'learning_rate': 7.363690733139842e-07, 'epoch': 0.1790073230268511, 'num_input_tokens_seen': 3061841920, 'completed': '39.60% (1_460 / 3_687)', 'remaining time': '19:51:13', 'throughput': '8555.99', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 21:21:25,288 >> {'loss': 0.7213, 'grad_norm': 22.699716567993164, 'learning_rate': 7.360015583640049e-07, 'epoch': 0.17927854624355846, 'num_input_tokens_seen': 3063939072, 'completed': '39.63% (1_461 / 3_687)', 'remaining time': '19:50:36', 'throughput': '8552.31', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 21:21:54,334 >> {'loss': 0.2935, 'grad_norm': 10.556961059570312, 'learning_rate': 7.356338937270492e-07, 'epoch': 0.1795497694602658, 'num_input_tokens_seen': 3066036224, 'completed': '39.65% (1_462 / 3_687)', 'remaining time': '19:49:53', 'throughput': '9024.96', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 21:22:23,705 >> {'loss': 0.3626, 'grad_norm': 21.715940475463867, 'learning_rate': 7.352660796989999e-07, 'epoch': 0.17982099267697316, 'num_input_tokens_seen': 3068133376, 'completed': '39.68% (1_463 / 3_687)', 'remaining time': '19:49:12', 'throughput': '8925.38', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 21:22:52,700 >> {'loss': 0.4329, 'grad_norm': 14.98128604888916, 'learning_rate': 7.348981165758595e-07, 'epoch': 0.1800922158936805, 'num_input_tokens_seen': 3070230528, 'completed': '39.71% (1_464 / 3_687)', 'remaining time': '19:48:30', 'throughput': '9041.10', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 21:23:23,041 >> {'loss': 0.4274, 'grad_norm': 13.6019868850708, 'learning_rate': 7.345300046537507e-07, 'epoch': 0.18036343911038785, 'num_input_tokens_seen': 3072327680, 'completed': '39.73% (1_465 / 3_687)', 'remaining time': '19:47:52', 'throughput': '8639.79', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 21:23:53,961 >> {'loss': 0.6069, 'grad_norm': 19.526018142700195, 'learning_rate': 7.341617442289159e-07, 'epoch': 0.18063466232709519, 'num_input_tokens_seen': 3074424832, 'completed': '39.76% (1_466 / 3_687)', 'remaining time': '19:47:16', 'throughput': '8478.00', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 21:24:26,387 >> {'loss': 0.484, 'grad_norm': 18.811630249023438, 'learning_rate': 7.337933355977175e-07, 'epoch': 0.18090588554380255, 'num_input_tokens_seen': 3076521984, 'completed': '39.79% (1_467 / 3_687)', 'remaining time': '19:46:45', 'throughput': '8084.57', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 21:24:57,639 >> {'loss': 0.3532, 'grad_norm': 11.840438842773438, 'learning_rate': 7.334247790566364e-07, 'epoch': 0.1811771087605099, 'num_input_tokens_seen': 3078619136, 'completed': '39.82% (1_468 / 3_687)', 'remaining time': '19:46:10', 'throughput': '8387.96', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 21:25:28,550 >> {'loss': 0.348, 'grad_norm': 13.839353561401367, 'learning_rate': 7.330560749022728e-07, 'epoch': 0.18144833197721724, 'num_input_tokens_seen': 3080716288, 'completed': '39.84% (1_469 / 3_687)', 'remaining time': '19:45:34', 'throughput': '8480.66', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 21:26:01,782 >> {'loss': 0.5346, 'grad_norm': 19.16689682006836, 'learning_rate': 7.326872234313459e-07, 'epoch': 0.1817195551939246, 'num_input_tokens_seen': 3082813440, 'completed': '39.87% (1_470 / 3_687)', 'remaining time': '19:45:06', 'throughput': '7888.39', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 21:26:33,950 >> {'loss': 0.3933, 'grad_norm': 14.058456420898438, 'learning_rate': 7.323182249406936e-07, 'epoch': 0.18199077841063194, 'num_input_tokens_seen': 3084910592, 'completed': '39.90% (1_471 / 3_687)', 'remaining time': '19:44:34', 'throughput': '8149.22', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 21:27:03,736 >> {'loss': 0.6038, 'grad_norm': 20.379125595092773, 'learning_rate': 7.319490797272714e-07, 'epoch': 0.1822620016273393, 'num_input_tokens_seen': 3087007744, 'completed': '39.92% (1_472 / 3_687)', 'remaining time': '19:43:55', 'throughput': '8800.73', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 21:27:37,011 >> {'loss': 0.459, 'grad_norm': 15.348321914672852, 'learning_rate': 7.315797880881535e-07, 'epoch': 0.18253322484404666, 'num_input_tokens_seen': 3089104896, 'completed': '39.95% (1_473 / 3_687)', 'remaining time': '19:43:27', 'throughput': '7878.06', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 21:28:07,424 >> {'loss': 0.4144, 'grad_norm': 12.275803565979004, 'learning_rate': 7.312103503205318e-07, 'epoch': 0.182804448060754, 'num_input_tokens_seen': 3091202048, 'completed': '39.98% (1_474 / 3_687)', 'remaining time': '19:42:49', 'throughput': '8619.43', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 21:28:39,041 >> {'loss': 0.1598, 'grad_norm': 9.52855110168457, 'learning_rate': 7.308407667217158e-07, 'epoch': 0.18307567127746135, 'num_input_tokens_seen': 3093299200, 'completed': '40.01% (1_475 / 3_687)', 'remaining time': '19:42:16', 'throughput': '8291.28', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 21:29:10,474 >> {'loss': 0.5072, 'grad_norm': 17.207202911376953, 'learning_rate': 7.304710375891323e-07, 'epoch': 0.1833468944941687, 'num_input_tokens_seen': 3095396352, 'completed': '40.03% (1_476 / 3_687)', 'remaining time': '19:41:41', 'throughput': '8339.97', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 21:29:42,454 >> {'loss': 0.7901, 'grad_norm': 21.52804946899414, 'learning_rate': 7.30101163220325e-07, 'epoch': 0.18361811771087605, 'num_input_tokens_seen': 3097493504, 'completed': '40.06% (1_477 / 3_687)', 'remaining time': '19:41:09', 'throughput': '8197.28', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 21:30:15,809 >> {'loss': 0.6151, 'grad_norm': 19.89445686340332, 'learning_rate': 7.29731143912955e-07, 'epoch': 0.1838893409275834, 'num_input_tokens_seen': 3099590656, 'completed': '40.09% (1_478 / 3_687)', 'remaining time': '19:40:41', 'throughput': '7858.96', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 21:30:48,046 >> {'loss': 0.4936, 'grad_norm': 15.77376651763916, 'learning_rate': 7.293609799647996e-07, 'epoch': 0.18416056414429074, 'num_input_tokens_seen': 3101687808, 'completed': '40.11% (1_479 / 3_687)', 'remaining time': '19:40:10', 'throughput': '8131.69', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 21:31:20,268 >> {'loss': 0.5218, 'grad_norm': 18.152956008911133, 'learning_rate': 7.289906716737528e-07, 'epoch': 0.1844317873609981, 'num_input_tokens_seen': 3103784960, 'completed': '40.14% (1_480 / 3_687)', 'remaining time': '19:39:38', 'throughput': '8135.65', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 21:31:53,199 >> {'loss': 0.3309, 'grad_norm': 13.07524585723877, 'learning_rate': 7.286202193378244e-07, 'epoch': 0.18470301057770544, 'num_input_tokens_seen': 3105882112, 'completed': '40.17% (1_481 / 3_687)', 'remaining time': '19:39:09', 'throughput': '7960.57', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 21:32:24,762 >> {'loss': 0.4835, 'grad_norm': 15.021199226379395, 'learning_rate': 7.282496232551406e-07, 'epoch': 0.1849742337944128, 'num_input_tokens_seen': 3107979264, 'completed': '40.20% (1_482 / 3_687)', 'remaining time': '19:38:35', 'throughput': '8305.21', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 21:32:55,497 >> {'loss': 0.5135, 'grad_norm': 18.78805923461914, 'learning_rate': 7.278788837239429e-07, 'epoch': 0.18524545701112016, 'num_input_tokens_seen': 3110076416, 'completed': '40.22% (1_483 / 3_687)', 'remaining time': '19:37:59', 'throughput': '8529.09', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 21:33:28,011 >> {'loss': 0.3278, 'grad_norm': 12.501900672912598, 'learning_rate': 7.275080010425883e-07, 'epoch': 0.1855166802278275, 'num_input_tokens_seen': 3112173568, 'completed': '40.25% (1_484 / 3_687)', 'remaining time': '19:37:28', 'throughput': '8062.58', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 21:33:56,138 >> {'loss': 0.5373, 'grad_norm': 15.580584526062012, 'learning_rate': 7.271369755095494e-07, 'epoch': 0.18578790344453486, 'num_input_tokens_seen': 3114270720, 'completed': '40.28% (1_485 / 3_687)', 'remaining time': '19:36:43', 'throughput': '9320.03', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 21:34:22,676 >> {'loss': 0.7682, 'grad_norm': 20.438100814819336, 'learning_rate': 7.267658074234128e-07, 'epoch': 0.1860591266612422, 'num_input_tokens_seen': 3116367872, 'completed': '40.30% (1_486 / 3_687)', 'remaining time': '19:35:54', 'throughput': '9877.96', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 21:34:53,672 >> {'loss': 0.5544, 'grad_norm': 19.1994686126709, 'learning_rate': 7.26394497082881e-07, 'epoch': 0.18633034987794955, 'num_input_tokens_seen': 3118465024, 'completed': '40.33% (1_487 / 3_687)', 'remaining time': '19:35:18', 'throughput': '8457.32', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 21:35:24,826 >> {'loss': 0.461, 'grad_norm': 14.906885147094727, 'learning_rate': 7.260230447867703e-07, 'epoch': 0.18660157309465691, 'num_input_tokens_seen': 3120562176, 'completed': '40.36% (1_488 / 3_687)', 'remaining time': '19:34:43', 'throughput': '8414.57', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 21:35:56,993 >> {'loss': 0.5265, 'grad_norm': 16.73325538635254, 'learning_rate': 7.256514508340114e-07, 'epoch': 0.18687279631136425, 'num_input_tokens_seen': 3122659328, 'completed': '40.39% (1_489 / 3_687)', 'remaining time': '19:34:11', 'throughput': '8149.48', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 21:36:25,301 >> {'loss': 0.6415, 'grad_norm': 22.4089412689209, 'learning_rate': 7.252797155236488e-07, 'epoch': 0.1871440195280716, 'num_input_tokens_seen': 3124756480, 'completed': '40.41% (1_490 / 3_687)', 'remaining time': '19:33:28', 'throughput': '9260.28', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 21:36:55,795 >> {'loss': 0.5622, 'grad_norm': 17.32809066772461, 'learning_rate': 7.249078391548409e-07, 'epoch': 0.18741524274477894, 'num_input_tokens_seen': 3126853632, 'completed': '40.44% (1_491 / 3_687)', 'remaining time': '19:32:51', 'throughput': '8596.66', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 21:37:28,067 >> {'loss': 0.2816, 'grad_norm': 10.146015167236328, 'learning_rate': 7.245358220268599e-07, 'epoch': 0.1876864659614863, 'num_input_tokens_seen': 3128950784, 'completed': '40.47% (1_492 / 3_687)', 'remaining time': '19:32:19', 'throughput': '8122.96', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 21:38:00,913 >> {'loss': 0.617, 'grad_norm': 24.50543975830078, 'learning_rate': 7.24163664439091e-07, 'epoch': 0.18795768917819367, 'num_input_tokens_seen': 3131047936, 'completed': '40.49% (1_493 / 3_687)', 'remaining time': '19:31:50', 'throughput': '7980.93', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 21:38:31,865 >> {'loss': 0.3164, 'grad_norm': 13.438515663146973, 'learning_rate': 7.237913666910322e-07, 'epoch': 0.188228912394901, 'num_input_tokens_seen': 3133145088, 'completed': '40.52% (1_494 / 3_687)', 'remaining time': '19:31:14', 'throughput': '8469.38', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 21:39:02,499 >> {'loss': 0.5909, 'grad_norm': 17.03264045715332, 'learning_rate': 7.234189290822947e-07, 'epoch': 0.18850013561160836, 'num_input_tokens_seen': 3135242240, 'completed': '40.55% (1_495 / 3_687)', 'remaining time': '19:30:38', 'throughput': '8557.38', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 21:39:31,580 >> {'loss': 0.4531, 'grad_norm': 14.248628616333008, 'learning_rate': 7.230463519126024e-07, 'epoch': 0.1887713588283157, 'num_input_tokens_seen': 3137339392, 'completed': '40.57% (1_496 / 3_687)', 'remaining time': '19:29:56', 'throughput': '9014.17', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 21:40:02,625 >> {'loss': 0.433, 'grad_norm': 16.5543212890625, 'learning_rate': 7.226736354817908e-07, 'epoch': 0.18904258204502306, 'num_input_tokens_seen': 3139436544, 'completed': '40.60% (1_497 / 3_687)', 'remaining time': '19:29:21', 'throughput': '8444.01', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 21:40:27,129 >> {'loss': 1.4103, 'grad_norm': 35.90945053100586, 'learning_rate': 7.223007800898082e-07, 'epoch': 0.18931380526173042, 'num_input_tokens_seen': 3141533696, 'completed': '40.63% (1_498 / 3_687)', 'remaining time': '19:28:25', 'throughput': '10698.28', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 21:40:55,959 >> {'loss': 0.6804, 'grad_norm': 19.54884910583496, 'learning_rate': 7.219277860367143e-07, 'epoch': 0.18958502847843775, 'num_input_tokens_seen': 3143630848, 'completed': '40.66% (1_499 / 3_687)', 'remaining time': '19:27:43', 'throughput': '9092.66', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 21:41:26,008 >> {'loss': 0.7872, 'grad_norm': 24.173274993896484, 'learning_rate': 7.215546536226805e-07, 'epoch': 0.1898562516951451, 'num_input_tokens_seen': 3145728000, 'completed': '40.68% (1_500 / 3_687)', 'remaining time': '19:27:05', 'throughput': '8723.85', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 21:41:53,619 >> {'loss': 0.4702, 'grad_norm': 15.859006881713867, 'learning_rate': 7.211813831479896e-07, 'epoch': 0.19012747491185245, 'num_input_tokens_seen': 3147825152, 'completed': '40.71% (1_501 / 3_687)', 'remaining time': '19:26:20', 'throughput': '9494.32', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 21:42:25,635 >> {'loss': 0.4567, 'grad_norm': 14.973697662353516, 'learning_rate': 7.208079749130356e-07, 'epoch': 0.1903986981285598, 'num_input_tokens_seen': 3149922304, 'completed': '40.74% (1_502 / 3_687)', 'remaining time': '19:25:48', 'throughput': '8187.85', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 21:42:52,441 >> {'loss': 0.5384, 'grad_norm': 17.608720779418945, 'learning_rate': 7.204344292183228e-07, 'epoch': 0.19066992134526717, 'num_input_tokens_seen': 3152019456, 'completed': '40.76% (1_503 / 3_687)', 'remaining time': '19:24:59', 'throughput': '9779.26', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 21:43:21,512 >> {'loss': 0.3972, 'grad_norm': 17.97117042541504, 'learning_rate': 7.200607463644673e-07, 'epoch': 0.1909411445619745, 'num_input_tokens_seen': 3154116608, 'completed': '40.79% (1_504 / 3_687)', 'remaining time': '19:24:18', 'throughput': '9017.54', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 21:43:53,415 >> {'loss': 0.235, 'grad_norm': 17.989160537719727, 'learning_rate': 7.196869266521941e-07, 'epoch': 0.19121236777868186, 'num_input_tokens_seen': 3156213760, 'completed': '40.82% (1_505 / 3_687)', 'remaining time': '19:23:46', 'throughput': '8216.67', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 21:44:27,477 >> {'loss': 0.5892, 'grad_norm': 14.863112449645996, 'learning_rate': 7.193129703823395e-07, 'epoch': 0.1914835909953892, 'num_input_tokens_seen': 3158310912, 'completed': '40.85% (1_506 / 3_687)', 'remaining time': '19:23:20', 'throughput': '7696.21', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 21:44:56,793 >> {'loss': 0.4773, 'grad_norm': 16.317642211914062, 'learning_rate': 7.189388778558491e-07, 'epoch': 0.19175481421209656, 'num_input_tokens_seen': 3160408064, 'completed': '40.87% (1_507 / 3_687)', 'remaining time': '19:22:40', 'throughput': '8942.12', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 21:45:26,619 >> {'loss': 0.3363, 'grad_norm': 12.433012962341309, 'learning_rate': 7.185646493737785e-07, 'epoch': 0.1920260374288039, 'num_input_tokens_seen': 3162505216, 'completed': '40.90% (1_508 / 3_687)', 'remaining time': '19:22:01', 'throughput': '8789.01', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 21:45:58,310 >> {'loss': 0.3374, 'grad_norm': 12.628132820129395, 'learning_rate': 7.181902852372924e-07, 'epoch': 0.19229726064551125, 'num_input_tokens_seen': 3164602368, 'completed': '40.93% (1_509 / 3_687)', 'remaining time': '19:21:28', 'throughput': '8271.86', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 21:46:28,587 >> {'loss': 0.5477, 'grad_norm': 18.057632446289062, 'learning_rate': 7.17815785747665e-07, 'epoch': 0.19256848386221861, 'num_input_tokens_seen': 3166699520, 'completed': '40.95% (1_510 / 3_687)', 'remaining time': '19:20:51', 'throughput': '8658.22', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 21:46:57,561 >> {'loss': 0.4914, 'grad_norm': 17.424755096435547, 'learning_rate': 7.174411512062789e-07, 'epoch': 0.19283970707892595, 'num_input_tokens_seen': 3168796672, 'completed': '40.98% (1_511 / 3_687)', 'remaining time': '19:20:10', 'throughput': '9047.53', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 21:47:26,636 >> {'loss': 0.5325, 'grad_norm': 15.669259071350098, 'learning_rate': 7.170663819146259e-07, 'epoch': 0.1931109302956333, 'num_input_tokens_seen': 3170893824, 'completed': '41.01% (1_512 / 3_687)', 'remaining time': '19:19:29', 'throughput': '9016.16', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 21:47:55,258 >> {'loss': 0.6616, 'grad_norm': 19.925724029541016, 'learning_rate': 7.166914781743062e-07, 'epoch': 0.19338215351234064, 'num_input_tokens_seen': 3172990976, 'completed': '41.04% (1_513 / 3_687)', 'remaining time': '19:18:47', 'throughput': '9158.89', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 21:48:26,781 >> {'loss': 0.3527, 'grad_norm': 12.23149585723877, 'learning_rate': 7.16316440287028e-07, 'epoch': 0.193653376729048, 'num_input_tokens_seen': 3175088128, 'completed': '41.06% (1_514 / 3_687)', 'remaining time': '19:18:13', 'throughput': '8315.87', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 21:48:58,365 >> {'loss': 0.5477, 'grad_norm': 20.181726455688477, 'learning_rate': 7.159412685546073e-07, 'epoch': 0.19392459994575537, 'num_input_tokens_seen': 3177185280, 'completed': '41.09% (1_515 / 3_687)', 'remaining time': '19:17:40', 'throughput': '8300.03', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 21:49:28,774 >> {'loss': 0.7233, 'grad_norm': 19.928884506225586, 'learning_rate': 7.155659632789683e-07, 'epoch': 0.1941958231624627, 'num_input_tokens_seen': 3179282432, 'completed': '41.12% (1_516 / 3_687)', 'remaining time': '19:17:03', 'throughput': '8620.44', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 21:50:01,697 >> {'loss': 0.6027, 'grad_norm': 20.663057327270508, 'learning_rate': 7.151905247621422e-07, 'epoch': 0.19446704637917006, 'num_input_tokens_seen': 3181379584, 'completed': '41.14% (1_517 / 3_687)', 'remaining time': '19:16:34', 'throughput': '7962.39', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 21:50:29,017 >> {'loss': 0.4609, 'grad_norm': 15.95964527130127, 'learning_rate': 7.148149533062678e-07, 'epoch': 0.1947382695958774, 'num_input_tokens_seen': 3183476736, 'completed': '41.17% (1_518 / 3_687)', 'remaining time': '19:15:48', 'throughput': '9595.41', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 21:50:58,815 >> {'loss': 0.4759, 'grad_norm': 18.054109573364258, 'learning_rate': 7.144392492135908e-07, 'epoch': 0.19500949281258476, 'num_input_tokens_seen': 3185573888, 'completed': '41.20% (1_519 / 3_687)', 'remaining time': '19:15:10', 'throughput': '8797.27', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 21:51:27,532 >> {'loss': 0.7295, 'grad_norm': 36.16549301147461, 'learning_rate': 7.140634127864632e-07, 'epoch': 0.19528071602929212, 'num_input_tokens_seen': 3187671040, 'completed': '41.23% (1_520 / 3_687)', 'remaining time': '19:14:28', 'throughput': '9128.54', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 21:51:57,950 >> {'loss': 0.4694, 'grad_norm': 16.543094635009766, 'learning_rate': 7.136874443273442e-07, 'epoch': 0.19555193924599945, 'num_input_tokens_seen': 3189768192, 'completed': '41.25% (1_521 / 3_687)', 'remaining time': '19:13:51', 'throughput': '8617.99', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 21:52:25,510 >> {'loss': 0.6975, 'grad_norm': 18.826963424682617, 'learning_rate': 7.133113441387988e-07, 'epoch': 0.1958231624627068, 'num_input_tokens_seen': 3191865344, 'completed': '41.28% (1_522 / 3_687)', 'remaining time': '19:13:06', 'throughput': '9511.91', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 21:52:57,351 >> {'loss': 0.2668, 'grad_norm': 11.468395233154297, 'learning_rate': 7.129351125234979e-07, 'epoch': 0.19609438567941415, 'num_input_tokens_seen': 3193962496, 'completed': '41.31% (1_523 / 3_687)', 'remaining time': '19:12:34', 'throughput': '8232.74', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 21:53:25,071 >> {'loss': 0.6557, 'grad_norm': 18.200729370117188, 'learning_rate': 7.125587497842189e-07, 'epoch': 0.1963656088961215, 'num_input_tokens_seen': 3196059648, 'completed': '41.33% (1_524 / 3_687)', 'remaining time': '19:11:49', 'throughput': '9456.96', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 21:53:56,504 >> {'loss': 0.2728, 'grad_norm': 11.098851203918457, 'learning_rate': 7.121822562238436e-07, 'epoch': 0.19663683211282887, 'num_input_tokens_seen': 3198156800, 'completed': '41.36% (1_525 / 3_687)', 'remaining time': '19:11:16', 'throughput': '8339.77', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 21:54:26,806 >> {'loss': 0.5025, 'grad_norm': 15.489395141601562, 'learning_rate': 7.118056321453601e-07, 'epoch': 0.1969080553295362, 'num_input_tokens_seen': 3200253952, 'completed': '41.39% (1_526 / 3_687)', 'remaining time': '19:10:39', 'throughput': '8651.07', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 21:54:57,282 >> {'loss': 0.4449, 'grad_norm': 14.967645645141602, 'learning_rate': 7.11428877851861e-07, 'epoch': 0.19717927854624356, 'num_input_tokens_seen': 3202351104, 'completed': '41.42% (1_527 / 3_687)', 'remaining time': '19:10:03', 'throughput': '8601.61', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 21:55:27,496 >> {'loss': 0.4, 'grad_norm': 15.567303657531738, 'learning_rate': 7.110519936465438e-07, 'epoch': 0.1974505017629509, 'num_input_tokens_seen': 3204448256, 'completed': '41.44% (1_528 / 3_687)', 'remaining time': '19:09:25', 'throughput': '8676.25', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 21:55:59,252 >> {'loss': 0.3363, 'grad_norm': 16.95789337158203, 'learning_rate': 7.106749798327106e-07, 'epoch': 0.19772172497965826, 'num_input_tokens_seen': 3206545408, 'completed': '41.47% (1_529 / 3_687)', 'remaining time': '19:08:53', 'throughput': '8255.09', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 21:56:30,653 >> {'loss': 0.4492, 'grad_norm': 16.923965454101562, 'learning_rate': 7.102978367137679e-07, 'epoch': 0.19799294819636562, 'num_input_tokens_seen': 3208642560, 'completed': '41.50% (1_530 / 3_687)', 'remaining time': '19:08:19', 'throughput': '8348.03', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 21:56:59,754 >> {'loss': 0.4185, 'grad_norm': 16.046384811401367, 'learning_rate': 7.099205645932258e-07, 'epoch': 0.19826417141307295, 'num_input_tokens_seen': 3210739712, 'completed': '41.52% (1_531 / 3_687)', 'remaining time': '19:07:39', 'throughput': '9008.24', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 21:57:28,494 >> {'loss': 0.6639, 'grad_norm': 20.16338348388672, 'learning_rate': 7.095431637746988e-07, 'epoch': 0.19853539462978032, 'num_input_tokens_seen': 3212836864, 'completed': '41.55% (1_532 / 3_687)', 'remaining time': '19:06:58', 'throughput': '9121.17', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 21:58:00,326 >> {'loss': 0.3112, 'grad_norm': 12.801714897155762, 'learning_rate': 7.091656345619047e-07, 'epoch': 0.19880661784648765, 'num_input_tokens_seen': 3214934016, 'completed': '41.58% (1_533 / 3_687)', 'remaining time': '19:06:26', 'throughput': '8235.20', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 21:58:31,568 >> {'loss': 0.4997, 'grad_norm': 14.88927936553955, 'learning_rate': 7.087879772586647e-07, 'epoch': 0.199077841063195, 'num_input_tokens_seen': 3217031168, 'completed': '41.61% (1_534 / 3_687)', 'remaining time': '19:05:52', 'throughput': '8390.82', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 21:59:04,076 >> {'loss': 0.5001, 'grad_norm': 20.41654396057129, 'learning_rate': 7.084101921689029e-07, 'epoch': 0.19934906427990237, 'num_input_tokens_seen': 3219128320, 'completed': '41.63% (1_535 / 3_687)', 'remaining time': '19:05:21', 'throughput': '8064.03', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 21:59:34,319 >> {'loss': 0.5536, 'grad_norm': 18.13739013671875, 'learning_rate': 7.080322795966462e-07, 'epoch': 0.1996202874966097, 'num_input_tokens_seen': 3221225472, 'completed': '41.66% (1_536 / 3_687)', 'remaining time': '19:04:44', 'throughput': '8667.84', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 22:00:05,399 >> {'loss': 0.4459, 'grad_norm': 15.031099319458008, 'learning_rate': 7.076542398460247e-07, 'epoch': 0.19989151071331707, 'num_input_tokens_seen': 3223322624, 'completed': '41.69% (1_537 / 3_687)', 'remaining time': '19:04:10', 'throughput': '8434.48', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 22:00:32,677 >> {'loss': 0.4325, 'grad_norm': 16.9238224029541, 'learning_rate': 7.0727607322127e-07, 'epoch': 0.2001627339300244, 'num_input_tokens_seen': 3225419776, 'completed': '41.71% (1_538 / 3_687)', 'remaining time': '19:03:25', 'throughput': '9610.16', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 22:01:03,316 >> {'loss': 0.5089, 'grad_norm': 17.309598922729492, 'learning_rate': 7.068977800267164e-07, 'epoch': 0.20043395714673176, 'num_input_tokens_seen': 3227516928, 'completed': '41.74% (1_539 / 3_687)', 'remaining time': '19:02:49', 'throughput': '8555.80', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 22:01:34,443 >> {'loss': 0.3359, 'grad_norm': 14.402442932128906, 'learning_rate': 7.065193605667999e-07, 'epoch': 0.20070518036343912, 'num_input_tokens_seen': 3229614080, 'completed': '41.77% (1_540 / 3_687)', 'remaining time': '19:02:15', 'throughput': '8421.88', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 22:02:07,606 >> {'loss': 0.3825, 'grad_norm': 12.946207046508789, 'learning_rate': 7.06140815146058e-07, 'epoch': 0.20097640358014646, 'num_input_tokens_seen': 3231711232, 'completed': '41.80% (1_541 / 3_687)', 'remaining time': '19:01:46', 'throughput': '7904.67', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 22:02:36,541 >> {'loss': 0.6832, 'grad_norm': 21.32681655883789, 'learning_rate': 7.057621440691296e-07, 'epoch': 0.20124762679685382, 'num_input_tokens_seen': 3233808384, 'completed': '41.82% (1_542 / 3_687)', 'remaining time': '19:01:06', 'throughput': '9059.65', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 22:03:08,328 >> {'loss': 0.5939, 'grad_norm': 18.230838775634766, 'learning_rate': 7.053833476407549e-07, 'epoch': 0.20151885001356115, 'num_input_tokens_seen': 3235905536, 'completed': '41.85% (1_543 / 3_687)', 'remaining time': '19:00:33', 'throughput': '8246.88', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 22:03:38,896 >> {'loss': 0.3152, 'grad_norm': 14.465167045593262, 'learning_rate': 7.050044261657748e-07, 'epoch': 0.2017900732302685, 'num_input_tokens_seen': 3238002688, 'completed': '41.88% (1_544 / 3_687)', 'remaining time': '18:59:58', 'throughput': '8575.78', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 22:04:08,887 >> {'loss': 0.2369, 'grad_norm': 11.428820610046387, 'learning_rate': 7.04625379949131e-07, 'epoch': 0.20206129644697587, 'num_input_tokens_seen': 3240099840, 'completed': '41.90% (1_545 / 3_687)', 'remaining time': '18:59:20', 'throughput': '8740.81', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 22:04:38,843 >> {'loss': 0.4465, 'grad_norm': 19.767881393432617, 'learning_rate': 7.042462092958651e-07, 'epoch': 0.2023325196636832, 'num_input_tokens_seen': 3242196992, 'completed': '41.93% (1_546 / 3_687)', 'remaining time': '18:58:43', 'throughput': '8751.18', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 22:05:08,612 >> {'loss': 0.4017, 'grad_norm': 13.485447883605957, 'learning_rate': 7.038669145111195e-07, 'epoch': 0.20260374288039057, 'num_input_tokens_seen': 3244294144, 'completed': '41.96% (1_547 / 3_687)', 'remaining time': '18:58:05', 'throughput': '8805.77', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 22:05:35,918 >> {'loss': 0.4824, 'grad_norm': 15.75723648071289, 'learning_rate': 7.034874959001363e-07, 'epoch': 0.2028749660970979, 'num_input_tokens_seen': 3246391296, 'completed': '41.99% (1_548 / 3_687)', 'remaining time': '18:57:20', 'throughput': '9600.22', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 22:06:07,100 >> {'loss': 0.3523, 'grad_norm': 13.907236099243164, 'learning_rate': 7.031079537682569e-07, 'epoch': 0.20314618931380526, 'num_input_tokens_seen': 3248488448, 'completed': '42.01% (1_549 / 3_687)', 'remaining time': '18:56:46', 'throughput': '8407.04', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 22:06:36,035 >> {'loss': 0.6136, 'grad_norm': 24.97413444519043, 'learning_rate': 7.027282884209227e-07, 'epoch': 0.20341741253051263, 'num_input_tokens_seen': 3250585600, 'completed': '42.04% (1_550 / 3_687)', 'remaining time': '18:56:05', 'throughput': '9059.65', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 22:07:08,197 >> {'loss': 0.3508, 'grad_norm': 15.245758056640625, 'learning_rate': 7.023485001636737e-07, 'epoch': 0.20368863574721996, 'num_input_tokens_seen': 3252682752, 'completed': '42.07% (1_551 / 3_687)', 'remaining time': '18:55:34', 'throughput': '8150.70', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 22:07:39,910 >> {'loss': 0.4695, 'grad_norm': 16.83147430419922, 'learning_rate': 7.019685893021488e-07, 'epoch': 0.20395985896392732, 'num_input_tokens_seen': 3254779904, 'completed': '42.09% (1_552 / 3_687)', 'remaining time': '18:55:02', 'throughput': '8266.03', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 22:08:09,734 >> {'loss': 0.2958, 'grad_norm': 17.15081787109375, 'learning_rate': 7.015885561420863e-07, 'epoch': 0.20423108218063465, 'num_input_tokens_seen': 3256877056, 'completed': '42.12% (1_553 / 3_687)', 'remaining time': '18:54:24', 'throughput': '8789.85', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 22:08:41,267 >> {'loss': 0.2439, 'grad_norm': 11.432106018066406, 'learning_rate': 7.012084009893221e-07, 'epoch': 0.20450230539734202, 'num_input_tokens_seen': 3258974208, 'completed': '42.15% (1_554 / 3_687)', 'remaining time': '18:53:51', 'throughput': '8313.40', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 22:09:10,762 >> {'loss': 0.604, 'grad_norm': 18.06707000732422, 'learning_rate': 7.008281241497908e-07, 'epoch': 0.20477352861404935, 'num_input_tokens_seen': 3261071360, 'completed': '42.18% (1_555 / 3_687)', 'remaining time': '18:53:12', 'throughput': '8887.98', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 22:09:41,068 >> {'loss': 0.4381, 'grad_norm': 17.187938690185547, 'learning_rate': 7.004477259295244e-07, 'epoch': 0.2050447518307567, 'num_input_tokens_seen': 3263168512, 'completed': '42.20% (1_556 / 3_687)', 'remaining time': '18:52:36', 'throughput': '8649.67', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 22:10:10,778 >> {'loss': 0.1795, 'grad_norm': 8.261337280273438, 'learning_rate': 7.000672066346532e-07, 'epoch': 0.20531597504746407, 'num_input_tokens_seen': 3265265664, 'completed': '42.23% (1_557 / 3_687)', 'remaining time': '18:51:58', 'throughput': '8823.30', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 22:10:41,821 >> {'loss': 0.319, 'grad_norm': 14.19255256652832, 'learning_rate': 6.996865665714046e-07, 'epoch': 0.2055871982641714, 'num_input_tokens_seen': 3267362816, 'completed': '42.26% (1_558 / 3_687)', 'remaining time': '18:51:24', 'throughput': '8445.35', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 22:11:12,730 >> {'loss': 0.5702, 'grad_norm': 21.178258895874023, 'learning_rate': 6.993058060461035e-07, 'epoch': 0.20585842148087877, 'num_input_tokens_seen': 3269459968, 'completed': '42.28% (1_559 / 3_687)', 'remaining time': '18:50:49', 'throughput': '8481.93', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 22:11:43,260 >> {'loss': 0.5158, 'grad_norm': 17.968137741088867, 'learning_rate': 6.989249253651708e-07, 'epoch': 0.2061296446975861, 'num_input_tokens_seen': 3271557120, 'completed': '42.31% (1_560 / 3_687)', 'remaining time': '18:50:13', 'throughput': '8584.74', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 22:12:17,525 >> {'loss': 0.2995, 'grad_norm': 12.363228797912598, 'learning_rate': 6.985439248351254e-07, 'epoch': 0.20640086791429346, 'num_input_tokens_seen': 3273654272, 'completed': '42.34% (1_561 / 3_687)', 'remaining time': '18:49:48', 'throughput': '7650.63', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 22:12:48,748 >> {'loss': 0.9089, 'grad_norm': 28.861257553100586, 'learning_rate': 6.981628047625818e-07, 'epoch': 0.20667209113100082, 'num_input_tokens_seen': 3275751424, 'completed': '42.37% (1_562 / 3_687)', 'remaining time': '18:49:14', 'throughput': '8395.88', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 22:13:19,864 >> {'loss': 0.5814, 'grad_norm': 19.901887893676758, 'learning_rate': 6.977815654542508e-07, 'epoch': 0.20694331434770816, 'num_input_tokens_seen': 3277848576, 'completed': '42.39% (1_563 / 3_687)', 'remaining time': '18:48:40', 'throughput': '8424.58', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 22:13:48,857 >> {'loss': 0.2016, 'grad_norm': 15.665919303894043, 'learning_rate': 6.974002072169395e-07, 'epoch': 0.20721453756441552, 'num_input_tokens_seen': 3279945728, 'completed': '42.42% (1_564 / 3_687)', 'remaining time': '18:48:00', 'throughput': '9041.81', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 22:14:22,556 >> {'loss': 0.3062, 'grad_norm': 19.99432945251465, 'learning_rate': 6.970187303575505e-07, 'epoch': 0.20748576078112285, 'num_input_tokens_seen': 3282042880, 'completed': '42.45% (1_565 / 3_687)', 'remaining time': '18:47:34', 'throughput': '7779.13', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 22:14:50,244 >> {'loss': 0.8428, 'grad_norm': 22.537017822265625, 'learning_rate': 6.966371351830818e-07, 'epoch': 0.2077569839978302, 'num_input_tokens_seen': 3284140032, 'completed': '42.47% (1_566 / 3_687)', 'remaining time': '18:46:50', 'throughput': '9467.61', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 22:15:22,905 >> {'loss': 0.3116, 'grad_norm': 13.440853118896484, 'learning_rate': 6.962554220006265e-07, 'epoch': 0.20802820721453757, 'num_input_tokens_seen': 3286237184, 'completed': '42.50% (1_567 / 3_687)', 'remaining time': '18:46:20', 'throughput': '8026.16', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 22:15:53,636 >> {'loss': 0.4781, 'grad_norm': 17.965656280517578, 'learning_rate': 6.958735911173729e-07, 'epoch': 0.2082994304312449, 'num_input_tokens_seen': 3288334336, 'completed': '42.53% (1_568 / 3_687)', 'remaining time': '18:45:45', 'throughput': '8530.34', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 22:16:23,573 >> {'loss': 0.6427, 'grad_norm': 19.18067741394043, 'learning_rate': 6.954916428406045e-07, 'epoch': 0.20857065364795227, 'num_input_tokens_seen': 3290431488, 'completed': '42.55% (1_569 / 3_687)', 'remaining time': '18:45:08', 'throughput': '8756.27', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 22:16:54,263 >> {'loss': 0.5277, 'grad_norm': 19.246889114379883, 'learning_rate': 6.95109577477698e-07, 'epoch': 0.2088418768646596, 'num_input_tokens_seen': 3292528640, 'completed': '42.58% (1_570 / 3_687)', 'remaining time': '18:44:33', 'throughput': '8541.75', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 22:17:22,873 >> {'loss': 0.5097, 'grad_norm': 16.653470993041992, 'learning_rate': 6.947273953361255e-07, 'epoch': 0.20911310008136696, 'num_input_tokens_seen': 3294625792, 'completed': '42.61% (1_571 / 3_687)', 'remaining time': '18:43:52', 'throughput': '9162.75', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 22:17:55,698 >> {'loss': 0.3398, 'grad_norm': 16.175172805786133, 'learning_rate': 6.943450967234524e-07, 'epoch': 0.20938432329807433, 'num_input_tokens_seen': 3296722944, 'completed': '42.64% (1_572 / 3_687)', 'remaining time': '18:43:23', 'throughput': '7986.01', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 22:18:24,755 >> {'loss': 0.3364, 'grad_norm': 15.780593872070312, 'learning_rate': 6.939626819473384e-07, 'epoch': 0.20965554651478166, 'num_input_tokens_seen': 3298820096, 'completed': '42.66% (1_573 / 3_687)', 'remaining time': '18:42:43', 'throughput': '9021.94', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 22:18:51,694 >> {'loss': 0.7428, 'grad_norm': 22.667606353759766, 'learning_rate': 6.93580151315536e-07, 'epoch': 0.20992676973148902, 'num_input_tokens_seen': 3300917248, 'completed': '42.69% (1_574 / 3_687)', 'remaining time': '18:41:58', 'throughput': '9730.92', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 22:19:20,552 >> {'loss': 0.3328, 'grad_norm': 15.267520904541016, 'learning_rate': 6.931975051358914e-07, 'epoch': 0.21019799294819635, 'num_input_tokens_seen': 3303014400, 'completed': '42.72% (1_575 / 3_687)', 'remaining time': '18:41:18', 'throughput': '9083.76', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 22:19:48,862 >> {'loss': 0.5853, 'grad_norm': 19.079408645629883, 'learning_rate': 6.928147437163439e-07, 'epoch': 0.21046921616490372, 'num_input_tokens_seen': 3305111552, 'completed': '42.74% (1_576 / 3_687)', 'remaining time': '18:40:37', 'throughput': '9259.75', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 22:20:16,599 >> {'loss': 0.3437, 'grad_norm': 13.351947784423828, 'learning_rate': 6.92431867364925e-07, 'epoch': 0.21074043938161108, 'num_input_tokens_seen': 3307208704, 'completed': '42.77% (1_577 / 3_687)', 'remaining time': '18:39:54', 'throughput': '9451.13', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 22:20:44,418 >> {'loss': 0.8318, 'grad_norm': 24.631118774414062, 'learning_rate': 6.920488763897593e-07, 'epoch': 0.2110116625983184, 'num_input_tokens_seen': 3309305856, 'completed': '42.80% (1_578 / 3_687)', 'remaining time': '18:39:11', 'throughput': '9423.30', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 22:21:16,556 >> {'loss': 0.5461, 'grad_norm': 17.947328567504883, 'learning_rate': 6.916657710990632e-07, 'epoch': 0.21128288581502577, 'num_input_tokens_seen': 3311403008, 'completed': '42.83% (1_579 / 3_687)', 'remaining time': '18:38:40', 'throughput': '8156.81', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 22:21:45,659 >> {'loss': 0.6746, 'grad_norm': 21.530193328857422, 'learning_rate': 6.912825518011452e-07, 'epoch': 0.2115541090317331, 'num_input_tokens_seen': 3313500160, 'completed': '42.85% (1_580 / 3_687)', 'remaining time': '18:38:01', 'throughput': '9007.45', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 22:22:15,560 >> {'loss': 0.3837, 'grad_norm': 16.4180850982666, 'learning_rate': 6.90899218804406e-07, 'epoch': 0.21182533224844047, 'num_input_tokens_seen': 3315597312, 'completed': '42.88% (1_581 / 3_687)', 'remaining time': '18:37:23', 'throughput': '8767.06', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 22:22:48,229 >> {'loss': 0.3731, 'grad_norm': 18.914493560791016, 'learning_rate': 6.905157724173369e-07, 'epoch': 0.21209655546514783, 'num_input_tokens_seen': 3317694464, 'completed': '42.91% (1_582 / 3_687)', 'remaining time': '18:36:54', 'throughput': '8024.23', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 22:23:18,118 >> {'loss': 0.4683, 'grad_norm': 15.548588752746582, 'learning_rate': 6.901322129485212e-07, 'epoch': 0.21236777868185516, 'num_input_tokens_seen': 3319791616, 'completed': '42.93% (1_583 / 3_687)', 'remaining time': '18:36:17', 'throughput': '8770.66', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 22:23:48,827 >> {'loss': 0.3252, 'grad_norm': 14.097211837768555, 'learning_rate': 6.897485407066329e-07, 'epoch': 0.21263900189856252, 'num_input_tokens_seen': 3321888768, 'completed': '42.96% (1_584 / 3_687)', 'remaining time': '18:35:42', 'throughput': '8536.37', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 22:24:21,721 >> {'loss': 0.5362, 'grad_norm': 15.844295501708984, 'learning_rate': 6.893647560004369e-07, 'epoch': 0.21291022511526986, 'num_input_tokens_seen': 3323985920, 'completed': '42.99% (1_585 / 3_687)', 'remaining time': '18:35:13', 'throughput': '7969.33', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 22:24:54,665 >> {'loss': 0.4111, 'grad_norm': 18.061901092529297, 'learning_rate': 6.889808591387885e-07, 'epoch': 0.21318144833197722, 'num_input_tokens_seen': 3326083072, 'completed': '43.02% (1_586 / 3_687)', 'remaining time': '18:34:44', 'throughput': '7957.20', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 22:25:25,384 >> {'loss': 0.408, 'grad_norm': 15.680930137634277, 'learning_rate': 6.885968504306334e-07, 'epoch': 0.21345267154868458, 'num_input_tokens_seen': 3328180224, 'completed': '43.04% (1_587 / 3_687)', 'remaining time': '18:34:09', 'throughput': '8533.81', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 22:25:55,408 >> {'loss': 0.684, 'grad_norm': 22.98466682434082, 'learning_rate': 6.882127301850069e-07, 'epoch': 0.21372389476539191, 'num_input_tokens_seen': 3330277376, 'completed': '43.07% (1_588 / 3_687)', 'remaining time': '18:33:33', 'throughput': '8731.01', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 22:26:25,616 >> {'loss': 0.576, 'grad_norm': 20.19873046875, 'learning_rate': 6.878284987110345e-07, 'epoch': 0.21399511798209928, 'num_input_tokens_seen': 3332374528, 'completed': '43.10% (1_589 / 3_687)', 'remaining time': '18:32:56', 'throughput': '8678.06', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 22:26:55,606 >> {'loss': 0.3008, 'grad_norm': 12.961610794067383, 'learning_rate': 6.874441563179313e-07, 'epoch': 0.2142663411988066, 'num_input_tokens_seen': 3334471680, 'completed': '43.12% (1_590 / 3_687)', 'remaining time': '18:32:20', 'throughput': '8741.01', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 22:27:25,466 >> {'loss': 0.6572, 'grad_norm': 22.00704574584961, 'learning_rate': 6.870597033150012e-07, 'epoch': 0.21453756441551397, 'num_input_tokens_seen': 3336568832, 'completed': '43.15% (1_591 / 3_687)', 'remaining time': '18:31:43', 'throughput': '8779.01', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 22:27:59,957 >> {'loss': 0.4674, 'grad_norm': 18.104747772216797, 'learning_rate': 6.866751400116374e-07, 'epoch': 0.21480878763222133, 'num_input_tokens_seen': 3338665984, 'completed': '43.18% (1_592 / 3_687)', 'remaining time': '18:31:18', 'throughput': '7600.47', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 22:28:29,972 >> {'loss': 0.7152, 'grad_norm': 24.767946243286133, 'learning_rate': 6.862904667173216e-07, 'epoch': 0.21508001084892867, 'num_input_tokens_seen': 3340763136, 'completed': '43.21% (1_593 / 3_687)', 'remaining time': '18:30:41', 'throughput': '8733.58', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 22:29:01,482 >> {'loss': 0.8405, 'grad_norm': 23.480615615844727, 'learning_rate': 6.859056837416245e-07, 'epoch': 0.21535123406563603, 'num_input_tokens_seen': 3342860288, 'completed': '43.23% (1_594 / 3_687)', 'remaining time': '18:30:09', 'throughput': '8319.48', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 22:29:30,663 >> {'loss': 0.3923, 'grad_norm': 13.278249740600586, 'learning_rate': 6.855207913942048e-07, 'epoch': 0.21562245728234336, 'num_input_tokens_seen': 3344957440, 'completed': '43.26% (1_595 / 3_687)', 'remaining time': '18:29:30', 'throughput': '8983.22', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 22:30:01,592 >> {'loss': 0.3845, 'grad_norm': 14.385771751403809, 'learning_rate': 6.85135789984809e-07, 'epoch': 0.21589368049905072, 'num_input_tokens_seen': 3347054592, 'completed': '43.29% (1_596 / 3_687)', 'remaining time': '18:28:56', 'throughput': '8475.79', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 22:30:30,753 >> {'loss': 0.5702, 'grad_norm': 15.939520835876465, 'learning_rate': 6.847506798232719e-07, 'epoch': 0.21616490371575806, 'num_input_tokens_seen': 3349151744, 'completed': '43.31% (1_597 / 3_687)', 'remaining time': '18:28:17', 'throughput': '8989.46', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 22:31:02,708 >> {'loss': 0.4505, 'grad_norm': 15.751465797424316, 'learning_rate': 6.843654612195152e-07, 'epoch': 0.21643612693246542, 'num_input_tokens_seen': 3351248896, 'completed': '43.34% (1_598 / 3_687)', 'remaining time': '18:27:45', 'throughput': '8203.63', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 22:31:32,590 >> {'loss': 0.5949, 'grad_norm': 16.361385345458984, 'learning_rate': 6.839801344835484e-07, 'epoch': 0.21670735014917278, 'num_input_tokens_seen': 3353346048, 'completed': '43.37% (1_599 / 3_687)', 'remaining time': '18:27:09', 'throughput': '8772.67', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 22:32:04,576 >> {'loss': 0.6065, 'grad_norm': 15.845523834228516, 'learning_rate': 6.835946999254677e-07, 'epoch': 0.2169785733658801, 'num_input_tokens_seen': 3355443200, 'completed': '43.40% (1_600 / 3_687)', 'remaining time': '18:26:37', 'throughput': '8195.59', 'gpu_mem_free': '5581MB'} +/scratch3/workspace/ctpham_umass_edu-ft/envs/prolong-final/lib/python3.10/site-packages/torch/distributed/fsdp/fully_sharded_data_parallel.py:689: FutureWarning: FSDP.state_dict_type() and FSDP.set_state_dict_type() are being deprecated. Please use APIs, get_state_dict() and set_state_dict(), which can support different parallelisms, FSDP1, FSDP2, DDP. API doc: https://pytorch.org/docs/stable/distributed.checkpoint.html#torch.distributed.checkpoint.state_dict.get_state_dict .Tutorial: https://pytorch.org/tutorials/recipes/distributed_checkpoint_recipe.html . + warnings.warn( +[INFO|trainer.py:3503] 2024-12-25 22:32:28,633 >> Saving model checkpoint to /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/checkpoint-1600 +[INFO|configuration_utils.py:472] 2024-12-25 22:32:28,636 >> Configuration saved in /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/checkpoint-1600/config.json +[INFO|configuration_utils.py:807] 2024-12-25 22:32:28,637 >> Configuration saved in /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/checkpoint-1600/generation_config.json +[INFO|modeling_utils.py:2807] 2024-12-25 22:33:30,637 >> The model is bigger than the maximum size per checkpoint (5GB) and is going to be split in 7 checkpoint shards. You can find where each parameters has been saved in the index located at /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/checkpoint-1600/model.safetensors.index.json. +[INFO|tokenization_utils_base.py:2684] 2024-12-25 22:33:30,641 >> tokenizer config file saved in /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/checkpoint-1600/tokenizer_config.json +[INFO|tokenization_utils_base.py:2693] 2024-12-25 22:33:30,642 >> Special tokens file saved in /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/checkpoint-1600/special_tokens_map.json +/scratch3/workspace/ctpham_umass_edu-ft/envs/prolong-final/lib/python3.10/site-packages/torch/distributed/fsdp/fully_sharded_data_parallel.py:689: FutureWarning: FSDP.state_dict_type() and FSDP.set_state_dict_type() are being deprecated. Please use APIs, get_state_dict() and set_state_dict(), which can support different parallelisms, FSDP1, FSDP2, DDP. API doc: https://pytorch.org/docs/stable/distributed.checkpoint.html#torch.distributed.checkpoint.state_dict.get_state_dict .Tutorial: https://pytorch.org/tutorials/recipes/distributed_checkpoint_recipe.html . + warnings.warn( +[WARNING|trainer.py:868] 2024-12-25 22:37:29,295 >> Save streaming dataset state: {'epoch': 0, 'sample_in_epoch': 3200, 'num_canonical_nodes': 1, 'shuffle_seed': 42, 'initial_physical_nodes': 1} +/scratch3/workspace/ctpham_umass_edu-ft/envs/prolong-final/lib/python3.10/site-packages/torch/utils/checkpoint.py:1399: FutureWarning: `torch.cpu.amp.autocast(args...)` is deprecated. Please use `torch.amp.autocast('cpu', args...)` instead. + with device_autocast_ctx, torch.cpu.amp.autocast(**cpu_autocast_kwargs), recompute_context: # type: ignore[attr-defined] +[INFO|trainer.py:175] 2024-12-25 22:37:59,455 >> {'loss': 0.4201, 'grad_norm': 13.730551719665527, 'learning_rate': 6.83209157855456e-07, 'epoch': 0.21724979658258747, 'num_input_tokens_seen': 3357540352, 'completed': '43.42% (1_601 / 3_687)', 'remaining time': '18:40:07', 'throughput': '738.69', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 22:38:31,391 >> {'loss': 0.4174, 'grad_norm': 17.727937698364258, 'learning_rate': 6.828235085837831e-07, 'epoch': 0.2175210197992948, 'num_input_tokens_seen': 3359637504, 'completed': '43.45% (1_602 / 3_687)', 'remaining time': '18:39:34', 'throughput': '8208.25', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 22:39:01,721 >> {'loss': 0.5056, 'grad_norm': 16.495952606201172, 'learning_rate': 6.824377524208047e-07, 'epoch': 0.21779224301600217, 'num_input_tokens_seen': 3361734656, 'completed': '43.48% (1_603 / 3_687)', 'remaining time': '18:38:57', 'throughput': '8643.24', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 22:39:32,963 >> {'loss': 0.4776, 'grad_norm': 16.65219497680664, 'learning_rate': 6.820518896769629e-07, 'epoch': 0.21806346623270953, 'num_input_tokens_seen': 3363831808, 'completed': '43.50% (1_604 / 3_687)', 'remaining time': '18:38:22', 'throughput': '8390.70', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 22:40:05,132 >> {'loss': 0.2099, 'grad_norm': 9.43620777130127, 'learning_rate': 6.816659206627853e-07, 'epoch': 0.21833468944941686, 'num_input_tokens_seen': 3365928960, 'completed': '43.53% (1_605 / 3_687)', 'remaining time': '18:37:50', 'throughput': '8148.87', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 22:40:37,364 >> {'loss': 0.4603, 'grad_norm': 16.026655197143555, 'learning_rate': 6.812798456888849e-07, 'epoch': 0.21860591266612422, 'num_input_tokens_seen': 3368026112, 'completed': '43.56% (1_606 / 3_687)', 'remaining time': '18:37:17', 'throughput': '8133.13', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 22:41:06,039 >> {'loss': 0.658, 'grad_norm': 18.902629852294922, 'learning_rate': 6.808936650659605e-07, 'epoch': 0.21887713588283156, 'num_input_tokens_seen': 3370123264, 'completed': '43.59% (1_607 / 3_687)', 'remaining time': '18:36:36', 'throughput': '9141.96', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 22:41:35,181 >> {'loss': 0.7047, 'grad_norm': 18.937278747558594, 'learning_rate': 6.805073791047951e-07, 'epoch': 0.21914835909953892, 'num_input_tokens_seen': 3372220416, 'completed': '43.61% (1_608 / 3_687)', 'remaining time': '18:35:56', 'throughput': '8995.28', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 22:42:05,128 >> {'loss': 0.303, 'grad_norm': 19.952560424804688, 'learning_rate': 6.80120988116257e-07, 'epoch': 0.21941958231624628, 'num_input_tokens_seen': 3374317568, 'completed': '43.64% (1_609 / 3_687)', 'remaining time': '18:35:18', 'throughput': '8753.63', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 22:42:37,905 >> {'loss': 0.3612, 'grad_norm': 16.27387046813965, 'learning_rate': 6.797344924112995e-07, 'epoch': 0.21969080553295361, 'num_input_tokens_seen': 3376414720, 'completed': '43.67% (1_610 / 3_687)', 'remaining time': '18:34:47', 'throughput': '7997.89', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 22:43:06,692 >> {'loss': 0.5804, 'grad_norm': 18.694629669189453, 'learning_rate': 6.793478923009592e-07, 'epoch': 0.21996202874966098, 'num_input_tokens_seen': 3378511872, 'completed': '43.69% (1_611 / 3_687)', 'remaining time': '18:34:06', 'throughput': '9106.20', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 22:43:37,436 >> {'loss': 0.2376, 'grad_norm': 11.021830558776855, 'learning_rate': 6.789611880963569e-07, 'epoch': 0.2202332519663683, 'num_input_tokens_seen': 3380609024, 'completed': '43.72% (1_612 / 3_687)', 'remaining time': '18:33:30', 'throughput': '8526.67', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 22:44:06,944 >> {'loss': 0.4313, 'grad_norm': 15.804499626159668, 'learning_rate': 6.78574380108698e-07, 'epoch': 0.22050447518307567, 'num_input_tokens_seen': 3382706176, 'completed': '43.75% (1_613 / 3_687)', 'remaining time': '18:32:51', 'throughput': '8883.99', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 22:44:35,342 >> {'loss': 0.4538, 'grad_norm': 15.294432640075684, 'learning_rate': 6.781874686492706e-07, 'epoch': 0.22077569839978303, 'num_input_tokens_seen': 3384803328, 'completed': '43.78% (1_614 / 3_687)', 'remaining time': '18:32:09', 'throughput': '9231.01', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 22:45:05,925 >> {'loss': 0.3465, 'grad_norm': 14.991973876953125, 'learning_rate': 6.778004540294464e-07, 'epoch': 0.22104692161649037, 'num_input_tokens_seen': 3386900480, 'completed': '43.80% (1_615 / 3_687)', 'remaining time': '18:31:33', 'throughput': '8571.49', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 22:45:32,432 >> {'loss': 0.7659, 'grad_norm': 22.537893295288086, 'learning_rate': 6.774133365606801e-07, 'epoch': 0.22131814483319773, 'num_input_tokens_seen': 3388997632, 'completed': '43.83% (1_616 / 3_687)', 'remaining time': '18:30:47', 'throughput': '9889.73', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 22:46:00,749 >> {'loss': 0.797, 'grad_norm': 21.337339401245117, 'learning_rate': 6.770261165545085e-07, 'epoch': 0.22158936804990506, 'num_input_tokens_seen': 3391094784, 'completed': '43.86% (1_617 / 3_687)', 'remaining time': '18:30:05', 'throughput': '9257.57', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 22:46:30,052 >> {'loss': 0.7643, 'grad_norm': 20.40045166015625, 'learning_rate': 6.766387943225524e-07, 'epoch': 0.22186059126661242, 'num_input_tokens_seen': 3393191936, 'completed': '43.88% (1_618 / 3_687)', 'remaining time': '18:29:25', 'throughput': '8945.75', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 22:46:57,876 >> {'loss': 0.409, 'grad_norm': 14.96789836883545, 'learning_rate': 6.762513701765135e-07, 'epoch': 0.22213181448331978, 'num_input_tokens_seen': 3395289088, 'completed': '43.91% (1_619 / 3_687)', 'remaining time': '18:28:42', 'throughput': '9421.67', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 22:47:30,523 >> {'loss': 0.4159, 'grad_norm': 14.952929496765137, 'learning_rate': 6.75863844428176e-07, 'epoch': 0.22240303770002712, 'num_input_tokens_seen': 3397386240, 'completed': '43.94% (1_620 / 3_687)', 'remaining time': '18:28:11', 'throughput': '8029.61', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 22:48:02,618 >> {'loss': 0.495, 'grad_norm': 16.563785552978516, 'learning_rate': 6.754762173894061e-07, 'epoch': 0.22267426091673448, 'num_input_tokens_seen': 3399483392, 'completed': '43.97% (1_621 / 3_687)', 'remaining time': '18:27:39', 'throughput': '8167.83', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 22:48:35,222 >> {'loss': 0.2923, 'grad_norm': 13.857934951782227, 'learning_rate': 6.750884893721511e-07, 'epoch': 0.2229454841334418, 'num_input_tokens_seen': 3401580544, 'completed': '43.99% (1_622 / 3_687)', 'remaining time': '18:27:08', 'throughput': '8040.21', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 22:49:07,590 >> {'loss': 0.4906, 'grad_norm': 16.87193489074707, 'learning_rate': 6.747006606884398e-07, 'epoch': 0.22321670735014917, 'num_input_tokens_seen': 3403677696, 'completed': '44.02% (1_623 / 3_687)', 'remaining time': '18:26:36', 'throughput': '8098.72', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 22:49:39,218 >> {'loss': 0.5691, 'grad_norm': 28.209985733032227, 'learning_rate': 6.74312731650382e-07, 'epoch': 0.22348793056685654, 'num_input_tokens_seen': 3405774848, 'completed': '44.05% (1_624 / 3_687)', 'remaining time': '18:26:02', 'throughput': '8288.34', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 22:50:11,747 >> {'loss': 0.4746, 'grad_norm': 18.54442596435547, 'learning_rate': 6.739247025701683e-07, 'epoch': 0.22375915378356387, 'num_input_tokens_seen': 3407872000, 'completed': '44.07% (1_625 / 3_687)', 'remaining time': '18:25:31', 'throughput': '8058.97', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 22:50:41,873 >> {'loss': 0.8189, 'grad_norm': 21.610734939575195, 'learning_rate': 6.735365737600695e-07, 'epoch': 0.22403037700027123, 'num_input_tokens_seen': 3409969152, 'completed': '44.10% (1_626 / 3_687)', 'remaining time': '18:24:54', 'throughput': '8702.57', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 22:51:13,741 >> {'loss': 0.3669, 'grad_norm': 13.132299423217773, 'learning_rate': 6.731483455324374e-07, 'epoch': 0.22430160021697856, 'num_input_tokens_seen': 3412066304, 'completed': '44.13% (1_627 / 3_687)', 'remaining time': '18:24:21', 'throughput': '8224.83', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 22:51:43,485 >> {'loss': 0.2336, 'grad_norm': 10.474742889404297, 'learning_rate': 6.727600181997026e-07, 'epoch': 0.22457282343368593, 'num_input_tokens_seen': 3414163456, 'completed': '44.16% (1_628 / 3_687)', 'remaining time': '18:23:43', 'throughput': '8813.38', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 22:52:11,498 >> {'loss': 0.5633, 'grad_norm': 20.9769287109375, 'learning_rate': 6.723715920743767e-07, 'epoch': 0.2248440466503933, 'num_input_tokens_seen': 3416260608, 'completed': '44.18% (1_629 / 3_687)', 'remaining time': '18:23:00', 'throughput': '9358.26', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 22:52:42,692 >> {'loss': 0.3815, 'grad_norm': 14.54977798461914, 'learning_rate': 6.7198306746905e-07, 'epoch': 0.22511526986710062, 'num_input_tokens_seen': 3418357760, 'completed': '44.21% (1_630 / 3_687)', 'remaining time': '18:22:26', 'throughput': '8403.44', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 22:53:11,593 >> {'loss': 0.5412, 'grad_norm': 19.554121017456055, 'learning_rate': 6.715944446963924e-07, 'epoch': 0.22538649308380798, 'num_input_tokens_seen': 3420454912, 'completed': '44.24% (1_631 / 3_687)', 'remaining time': '18:21:46', 'throughput': '9071.76', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 22:53:40,698 >> {'loss': 0.9327, 'grad_norm': 23.622020721435547, 'learning_rate': 6.712057240691527e-07, 'epoch': 0.22565771630051532, 'num_input_tokens_seen': 3422552064, 'completed': '44.26% (1_632 / 3_687)', 'remaining time': '18:21:06', 'throughput': '9005.52', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 22:54:10,728 >> {'loss': 0.3727, 'grad_norm': 13.846428871154785, 'learning_rate': 6.708169059001586e-07, 'epoch': 0.22592893951722268, 'num_input_tokens_seen': 3424649216, 'completed': '44.29% (1_633 / 3_687)', 'remaining time': '18:20:29', 'throughput': '8729.55', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 22:54:41,433 >> {'loss': 0.5854, 'grad_norm': 20.11954689025879, 'learning_rate': 6.704279905023159e-07, 'epoch': 0.22620016273393004, 'num_input_tokens_seen': 3426746368, 'completed': '44.32% (1_634 / 3_687)', 'remaining time': '18:19:53', 'throughput': '8537.28', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 22:55:12,407 >> {'loss': 0.6741, 'grad_norm': 28.113088607788086, 'learning_rate': 6.700389781886091e-07, 'epoch': 0.22647138595063737, 'num_input_tokens_seen': 3428843520, 'completed': '44.34% (1_635 / 3_687)', 'remaining time': '18:19:18', 'throughput': '8463.38', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 22:55:41,699 >> {'loss': 0.3548, 'grad_norm': 12.346856117248535, 'learning_rate': 6.696498692721006e-07, 'epoch': 0.22674260916734473, 'num_input_tokens_seen': 3430940672, 'completed': '44.37% (1_636 / 3_687)', 'remaining time': '18:18:39', 'throughput': '8949.57', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 22:56:13,973 >> {'loss': 0.5486, 'grad_norm': 16.389089584350586, 'learning_rate': 6.692606640659302e-07, 'epoch': 0.22701383238405207, 'num_input_tokens_seen': 3433037824, 'completed': '44.40% (1_637 / 3_687)', 'remaining time': '18:18:07', 'throughput': '8122.35', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 22:56:43,623 >> {'loss': 0.7321, 'grad_norm': 24.49221420288086, 'learning_rate': 6.688713628833157e-07, 'epoch': 0.22728505560075943, 'num_input_tokens_seen': 3435134976, 'completed': '44.43% (1_638 / 3_687)', 'remaining time': '18:17:29', 'throughput': '8841.27', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 22:57:14,200 >> {'loss': 0.4644, 'grad_norm': 16.724529266357422, 'learning_rate': 6.684819660375516e-07, 'epoch': 0.22755627881746676, 'num_input_tokens_seen': 3437232128, 'completed': '44.45% (1_639 / 3_687)', 'remaining time': '18:16:53', 'throughput': '8573.16', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 22:57:46,634 >> {'loss': 0.6855, 'grad_norm': 20.410425186157227, 'learning_rate': 6.6809247384201e-07, 'epoch': 0.22782750203417412, 'num_input_tokens_seen': 3439329280, 'completed': '44.48% (1_640 / 3_687)', 'remaining time': '18:16:21', 'throughput': '8082.50', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 22:58:19,112 >> {'loss': 0.4161, 'grad_norm': 14.939435005187988, 'learning_rate': 6.67702886610139e-07, 'epoch': 0.22809872525088148, 'num_input_tokens_seen': 3441426432, 'completed': '44.51% (1_641 / 3_687)', 'remaining time': '18:15:50', 'throughput': '8071.33', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 22:58:47,776 >> {'loss': 0.6421, 'grad_norm': 16.925172805786133, 'learning_rate': 6.673132046554639e-07, 'epoch': 0.22836994846758882, 'num_input_tokens_seen': 3443523584, 'completed': '44.53% (1_642 / 3_687)', 'remaining time': '18:15:09', 'throughput': '9145.49', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 22:59:17,837 >> {'loss': 0.2435, 'grad_norm': 11.812236785888672, 'learning_rate': 6.669234282915857e-07, 'epoch': 0.22864117168429618, 'num_input_tokens_seen': 3445620736, 'completed': '44.56% (1_643 / 3_687)', 'remaining time': '18:14:32', 'throughput': '8720.30', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 22:59:49,986 >> {'loss': 0.4875, 'grad_norm': 16.797849655151367, 'learning_rate': 6.665335578321819e-07, 'epoch': 0.2289123949010035, 'num_input_tokens_seen': 3447717888, 'completed': '44.59% (1_644 / 3_687)', 'remaining time': '18:14:00', 'throughput': '8154.17', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 23:00:19,280 >> {'loss': 0.6485, 'grad_norm': 20.539812088012695, 'learning_rate': 6.661435935910048e-07, 'epoch': 0.22918361811771087, 'num_input_tokens_seen': 3449815040, 'completed': '44.62% (1_645 / 3_687)', 'remaining time': '18:13:21', 'throughput': '8948.79', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 23:00:49,159 >> {'loss': 0.3879, 'grad_norm': 12.565629959106445, 'learning_rate': 6.657535358818833e-07, 'epoch': 0.22945484133441824, 'num_input_tokens_seen': 3451912192, 'completed': '44.64% (1_646 / 3_687)', 'remaining time': '18:12:44', 'throughput': '8773.38', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 23:01:19,434 >> {'loss': 0.6757, 'grad_norm': 21.272024154663086, 'learning_rate': 6.653633850187211e-07, 'epoch': 0.22972606455112557, 'num_input_tokens_seen': 3454009344, 'completed': '44.67% (1_647 / 3_687)', 'remaining time': '18:12:07', 'throughput': '8658.82', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 23:01:50,137 >> {'loss': 0.4601, 'grad_norm': 17.750259399414062, 'learning_rate': 6.649731413154964e-07, 'epoch': 0.22999728776783293, 'num_input_tokens_seen': 3456106496, 'completed': '44.70% (1_648 / 3_687)', 'remaining time': '18:11:32', 'throughput': '8537.94', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 23:02:18,932 >> {'loss': 0.5663, 'grad_norm': 17.899972915649414, 'learning_rate': 6.645828050862626e-07, 'epoch': 0.23026851098454026, 'num_input_tokens_seen': 3458203648, 'completed': '44.72% (1_649 / 3_687)', 'remaining time': '18:10:51', 'throughput': '9104.04', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 23:02:52,955 >> {'loss': 0.1971, 'grad_norm': 10.090818405151367, 'learning_rate': 6.641923766451475e-07, 'epoch': 0.23053973420124763, 'num_input_tokens_seen': 3460300800, 'completed': '44.75% (1_650 / 3_687)', 'remaining time': '18:10:24', 'throughput': '7704.76', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 23:03:24,660 >> {'loss': 0.4693, 'grad_norm': 14.428431510925293, 'learning_rate': 6.63801856306353e-07, 'epoch': 0.230810957417955, 'num_input_tokens_seen': 3462397952, 'completed': '44.78% (1_651 / 3_687)', 'remaining time': '18:09:51', 'throughput': '8268.31', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 23:03:53,814 >> {'loss': 0.5592, 'grad_norm': 18.0393123626709, 'learning_rate': 6.634112443841551e-07, 'epoch': 0.23108218063466232, 'num_input_tokens_seen': 3464495104, 'completed': '44.81% (1_652 / 3_687)', 'remaining time': '18:09:12', 'throughput': '8991.68', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 23:04:23,777 >> {'loss': 0.5662, 'grad_norm': 15.629161834716797, 'learning_rate': 6.630205411929032e-07, 'epoch': 0.23135340385136968, 'num_input_tokens_seen': 3466592256, 'completed': '44.83% (1_653 / 3_687)', 'remaining time': '18:08:34', 'throughput': '8749.04', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 23:04:55,329 >> {'loss': 0.3379, 'grad_norm': 15.491093635559082, 'learning_rate': 6.626297470470205e-07, 'epoch': 0.23162462706807702, 'num_input_tokens_seen': 3468689408, 'completed': '44.86% (1_654 / 3_687)', 'remaining time': '18:08:01', 'throughput': '8308.15', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 23:05:30,382 >> {'loss': 0.2769, 'grad_norm': 13.871119499206543, 'learning_rate': 6.622388622610034e-07, 'epoch': 0.23189585028478438, 'num_input_tokens_seen': 3470786560, 'completed': '44.89% (1_655 / 3_687)', 'remaining time': '18:07:36', 'throughput': '7478.58', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 23:06:01,520 >> {'loss': 0.3845, 'grad_norm': 12.842281341552734, 'learning_rate': 6.618478871494209e-07, 'epoch': 0.23216707350149174, 'num_input_tokens_seen': 3472883712, 'completed': '44.91% (1_656 / 3_687)', 'remaining time': '18:07:01', 'throughput': '8418.62', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 23:06:35,294 >> {'loss': 0.3783, 'grad_norm': 14.781960487365723, 'learning_rate': 6.61456822026915e-07, 'epoch': 0.23243829671819907, 'num_input_tokens_seen': 3474980864, 'completed': '44.94% (1_657 / 3_687)', 'remaining time': '18:06:33', 'throughput': '7761.78', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 23:07:05,529 >> {'loss': 0.5104, 'grad_norm': 18.07168960571289, 'learning_rate': 6.610656672081999e-07, 'epoch': 0.23270951993490643, 'num_input_tokens_seen': 3477078016, 'completed': '44.97% (1_658 / 3_687)', 'remaining time': '18:05:57', 'throughput': '8670.25', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 23:07:35,666 >> {'loss': 0.5309, 'grad_norm': 19.128738403320312, 'learning_rate': 6.606744230080622e-07, 'epoch': 0.23298074315161377, 'num_input_tokens_seen': 3479175168, 'completed': '45.00% (1_659 / 3_687)', 'remaining time': '18:05:20', 'throughput': '8698.54', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 23:08:03,181 >> {'loss': 0.7695, 'grad_norm': 21.630807876586914, 'learning_rate': 6.602830897413603e-07, 'epoch': 0.23325196636832113, 'num_input_tokens_seen': 3481272320, 'completed': '45.02% (1_660 / 3_687)', 'remaining time': '18:04:37', 'throughput': '9527.31', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 23:08:29,913 >> {'loss': 0.8973, 'grad_norm': 22.350727081298828, 'learning_rate': 6.598916677230243e-07, 'epoch': 0.2335231895850285, 'num_input_tokens_seen': 3483369472, 'completed': '45.05% (1_661 / 3_687)', 'remaining time': '18:03:52', 'throughput': '9806.17', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 23:09:00,301 >> {'loss': 0.4983, 'grad_norm': 16.569183349609375, 'learning_rate': 6.59500157268056e-07, 'epoch': 0.23379441280173582, 'num_input_tokens_seen': 3485466624, 'completed': '45.08% (1_662 / 3_687)', 'remaining time': '18:03:16', 'throughput': '8626.69', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 23:09:30,375 >> {'loss': 0.4791, 'grad_norm': 17.87363052368164, 'learning_rate': 6.591085586915279e-07, 'epoch': 0.23406563601844319, 'num_input_tokens_seen': 3487563776, 'completed': '45.10% (1_663 / 3_687)', 'remaining time': '18:02:39', 'throughput': '8716.67', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 23:10:01,386 >> {'loss': 0.7941, 'grad_norm': 22.503154754638672, 'learning_rate': 6.587168723085836e-07, 'epoch': 0.23433685923515052, 'num_input_tokens_seen': 3489660928, 'completed': '45.13% (1_664 / 3_687)', 'remaining time': '18:02:05', 'throughput': '8453.20', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 23:10:30,768 >> {'loss': 0.4266, 'grad_norm': 16.588794708251953, 'learning_rate': 6.583250984344374e-07, 'epoch': 0.23460808245185788, 'num_input_tokens_seen': 3491758080, 'completed': '45.16% (1_665 / 3_687)', 'remaining time': '18:01:26', 'throughput': '8921.94', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 23:11:00,000 >> {'loss': 0.3887, 'grad_norm': 20.612022399902344, 'learning_rate': 6.57933237384374e-07, 'epoch': 0.23487930566856524, 'num_input_tokens_seen': 3493855232, 'completed': '45.19% (1_666 / 3_687)', 'remaining time': '18:00:47', 'throughput': '8967.66', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 23:11:31,626 >> {'loss': 0.655, 'grad_norm': 19.953716278076172, 'learning_rate': 6.575412894737484e-07, 'epoch': 0.23515052888527258, 'num_input_tokens_seen': 3495952384, 'completed': '45.21% (1_667 / 3_687)', 'remaining time': '18:00:14', 'throughput': '8288.91', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 23:11:58,337 >> {'loss': 0.8792, 'grad_norm': 21.730518341064453, 'learning_rate': 6.571492550179853e-07, 'epoch': 0.23542175210197994, 'num_input_tokens_seen': 3498049536, 'completed': '45.24% (1_668 / 3_687)', 'remaining time': '17:59:30', 'throughput': '9813.88', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 23:12:27,522 >> {'loss': 0.6458, 'grad_norm': 20.50918197631836, 'learning_rate': 6.567571343325791e-07, 'epoch': 0.23569297531868727, 'num_input_tokens_seen': 3500146688, 'completed': '45.27% (1_669 / 3_687)', 'remaining time': '17:58:51', 'throughput': '8982.31', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 23:12:54,946 >> {'loss': 0.5007, 'grad_norm': 18.293838500976562, 'learning_rate': 6.563649277330935e-07, 'epoch': 0.23596419853539463, 'num_input_tokens_seen': 3502243840, 'completed': '45.29% (1_670 / 3_687)', 'remaining time': '17:58:08', 'throughput': '9558.70', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 23:13:26,913 >> {'loss': 0.4134, 'grad_norm': 13.126777648925781, 'learning_rate': 6.559726355351617e-07, 'epoch': 0.236235421752102, 'num_input_tokens_seen': 3504340992, 'completed': '45.32% (1_671 / 3_687)', 'remaining time': '17:57:36', 'throughput': '8200.44', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 23:13:55,937 >> {'loss': 0.345, 'grad_norm': 12.992348670959473, 'learning_rate': 6.555802580544853e-07, 'epoch': 0.23650664496880933, 'num_input_tokens_seen': 3506438144, 'completed': '45.35% (1_672 / 3_687)', 'remaining time': '17:56:57', 'throughput': '9032.16', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 23:14:28,735 >> {'loss': 0.413, 'grad_norm': 15.970709800720215, 'learning_rate': 6.551877956068349e-07, 'epoch': 0.2367778681855167, 'num_input_tokens_seen': 3508535296, 'completed': '45.38% (1_673 / 3_687)', 'remaining time': '17:56:26', 'throughput': '7992.67', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 23:14:57,936 >> {'loss': 0.5876, 'grad_norm': 19.215749740600586, 'learning_rate': 6.547952485080491e-07, 'epoch': 0.23704909140222402, 'num_input_tokens_seen': 3510632448, 'completed': '45.40% (1_674 / 3_687)', 'remaining time': '17:55:47', 'throughput': '8977.27', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 23:15:32,950 >> {'loss': 0.2881, 'grad_norm': 10.450196266174316, 'learning_rate': 6.544026170740352e-07, 'epoch': 0.23732031461893138, 'num_input_tokens_seen': 3512729600, 'completed': '45.43% (1_675 / 3_687)', 'remaining time': '17:55:22', 'throughput': '7486.79', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 23:16:05,376 >> {'loss': 0.3993, 'grad_norm': 13.23245906829834, 'learning_rate': 6.540099016207679e-07, 'epoch': 0.23759153783563874, 'num_input_tokens_seen': 3514826752, 'completed': '45.46% (1_676 / 3_687)', 'remaining time': '17:54:51', 'throughput': '8084.37', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 23:16:35,987 >> {'loss': 0.41, 'grad_norm': 13.348737716674805, 'learning_rate': 6.536171024642896e-07, 'epoch': 0.23786276105234608, 'num_input_tokens_seen': 3516923904, 'completed': '45.48% (1_677 / 3_687)', 'remaining time': '17:54:16', 'throughput': '8563.68', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 23:17:06,924 >> {'loss': 0.3554, 'grad_norm': 15.210914611816406, 'learning_rate': 6.532242199207103e-07, 'epoch': 0.23813398426905344, 'num_input_tokens_seen': 3519021056, 'completed': '45.51% (1_678 / 3_687)', 'remaining time': '17:53:41', 'throughput': '8473.51', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 23:17:35,964 >> {'loss': 0.2304, 'grad_norm': 14.020462989807129, 'learning_rate': 6.528312543062066e-07, 'epoch': 0.23840520748576077, 'num_input_tokens_seen': 3521118208, 'completed': '45.54% (1_679 / 3_687)', 'remaining time': '17:53:02', 'throughput': '9027.13', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 23:18:04,327 >> {'loss': 0.5624, 'grad_norm': 18.36133575439453, 'learning_rate': 6.524382059370226e-07, 'epoch': 0.23867643070246813, 'num_input_tokens_seen': 3523215360, 'completed': '45.57% (1_680 / 3_687)', 'remaining time': '17:52:21', 'throughput': '9242.32', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 23:18:33,330 >> {'loss': 0.5282, 'grad_norm': 16.818580627441406, 'learning_rate': 6.520450751294685e-07, 'epoch': 0.23894765391917547, 'num_input_tokens_seen': 3525312512, 'completed': '45.59% (1_681 / 3_687)', 'remaining time': '17:51:42', 'throughput': '9038.65', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 23:19:02,093 >> {'loss': 0.6669, 'grad_norm': 25.353300094604492, 'learning_rate': 6.516518621999209e-07, 'epoch': 0.23921887713588283, 'num_input_tokens_seen': 3527409664, 'completed': '45.62% (1_682 / 3_687)', 'remaining time': '17:51:03', 'throughput': '9113.95', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 23:19:33,164 >> {'loss': 0.6045, 'grad_norm': 18.05834197998047, 'learning_rate': 6.512585674648227e-07, 'epoch': 0.2394901003525902, 'num_input_tokens_seen': 3529506816, 'completed': '45.65% (1_683 / 3_687)', 'remaining time': '17:50:29', 'throughput': '8437.39', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 23:20:03,968 >> {'loss': 0.3037, 'grad_norm': 11.763015747070312, 'learning_rate': 6.50865191240683e-07, 'epoch': 0.23976132356929752, 'num_input_tokens_seen': 3531603968, 'completed': '45.67% (1_684 / 3_687)', 'remaining time': '17:49:54', 'throughput': '8509.91', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 23:20:33,641 >> {'loss': 0.5238, 'grad_norm': 19.146381378173828, 'learning_rate': 6.504717338440751e-07, 'epoch': 0.24003254678600489, 'num_input_tokens_seen': 3533701120, 'completed': '45.70% (1_685 / 3_687)', 'remaining time': '17:49:16', 'throughput': '8834.04', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 23:21:03,869 >> {'loss': 0.3522, 'grad_norm': 13.004151344299316, 'learning_rate': 6.500781955916393e-07, 'epoch': 0.24030377000271222, 'num_input_tokens_seen': 3535798272, 'completed': '45.73% (1_686 / 3_687)', 'remaining time': '17:48:40', 'throughput': '8672.12', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 23:21:32,974 >> {'loss': 0.5728, 'grad_norm': 17.20205307006836, 'learning_rate': 6.4968457680008e-07, 'epoch': 0.24057499321941958, 'num_input_tokens_seen': 3537895424, 'completed': '45.76% (1_687 / 3_687)', 'remaining time': '17:48:01', 'throughput': '9007.01', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 23:22:05,330 >> {'loss': 0.3135, 'grad_norm': 12.467523574829102, 'learning_rate': 6.492908777861664e-07, 'epoch': 0.24084621643612694, 'num_input_tokens_seen': 3539992576, 'completed': '45.78% (1_688 / 3_687)', 'remaining time': '17:47:30', 'throughput': '8101.82', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 23:22:34,829 >> {'loss': 0.6591, 'grad_norm': 21.274076461791992, 'learning_rate': 6.488970988667327e-07, 'epoch': 0.24111743965283428, 'num_input_tokens_seen': 3542089728, 'completed': '45.81% (1_689 / 3_687)', 'remaining time': '17:46:52', 'throughput': '8886.53', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 23:23:01,492 >> {'loss': 0.927, 'grad_norm': 24.0526180267334, 'learning_rate': 6.485032403586772e-07, 'epoch': 0.24138866286954164, 'num_input_tokens_seen': 3544186880, 'completed': '45.84% (1_690 / 3_687)', 'remaining time': '17:46:08', 'throughput': '9831.86', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 23:23:30,515 >> {'loss': 0.3657, 'grad_norm': 16.187849044799805, 'learning_rate': 6.481093025789621e-07, 'epoch': 0.24165988608624897, 'num_input_tokens_seen': 3546284032, 'completed': '45.86% (1_691 / 3_687)', 'remaining time': '17:45:29', 'throughput': '9032.05', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 23:24:00,466 >> {'loss': 0.5053, 'grad_norm': 25.65354347229004, 'learning_rate': 6.477152858446136e-07, 'epoch': 0.24193110930295633, 'num_input_tokens_seen': 3548381184, 'completed': '45.89% (1_692 / 3_687)', 'remaining time': '17:44:53', 'throughput': '8752.54', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 23:24:30,496 >> {'loss': 0.5336, 'grad_norm': 16.25813865661621, 'learning_rate': 6.473211904727216e-07, 'epoch': 0.2422023325196637, 'num_input_tokens_seen': 3550478336, 'completed': '45.92% (1_693 / 3_687)', 'remaining time': '17:44:16', 'throughput': '8729.51', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 23:25:00,872 >> {'loss': 0.5435, 'grad_norm': 19.955923080444336, 'learning_rate': 6.469270167804386e-07, 'epoch': 0.24247355573637103, 'num_input_tokens_seen': 3552575488, 'completed': '45.95% (1_694 / 3_687)', 'remaining time': '17:43:41', 'throughput': '8629.71', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 23:25:33,829 >> {'loss': 0.3573, 'grad_norm': 12.152280807495117, 'learning_rate': 6.46532765084981e-07, 'epoch': 0.2427447789530784, 'num_input_tokens_seen': 3554672640, 'completed': '45.97% (1_695 / 3_687)', 'remaining time': '17:43:11', 'throughput': '7954.14', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 23:26:06,242 >> {'loss': 0.2561, 'grad_norm': 13.028290748596191, 'learning_rate': 6.461384357036274e-07, 'epoch': 0.24301600216978572, 'num_input_tokens_seen': 3556769792, 'completed': '46.00% (1_696 / 3_687)', 'remaining time': '17:42:40', 'throughput': '8087.70', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 23:26:37,666 >> {'loss': 0.4922, 'grad_norm': 20.247438430786133, 'learning_rate': 6.457440289537191e-07, 'epoch': 0.24328722538649308, 'num_input_tokens_seen': 3558866944, 'completed': '46.03% (1_697 / 3_687)', 'remaining time': '17:42:06', 'throughput': '8342.22', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 23:27:07,987 >> {'loss': 0.301, 'grad_norm': 12.755880355834961, 'learning_rate': 6.453495451526595e-07, 'epoch': 0.24355844860320044, 'num_input_tokens_seen': 3560964096, 'completed': '46.05% (1_698 / 3_687)', 'remaining time': '17:41:30', 'throughput': '8645.43', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 23:27:40,148 >> {'loss': 0.3143, 'grad_norm': 13.987512588500977, 'learning_rate': 6.449549846179144e-07, 'epoch': 0.24382967181990778, 'num_input_tokens_seen': 3563061248, 'completed': '46.08% (1_699 / 3_687)', 'remaining time': '17:40:59', 'throughput': '8151.07', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 23:28:12,977 >> {'loss': 0.5411, 'grad_norm': 26.919103622436523, 'learning_rate': 6.445603476670109e-07, 'epoch': 0.24410089503661514, 'num_input_tokens_seen': 3565158400, 'completed': '46.11% (1_700 / 3_687)', 'remaining time': '17:40:28', 'throughput': '7985.14', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 23:28:40,416 >> {'loss': 0.7823, 'grad_norm': 21.93929672241211, 'learning_rate': 6.44165634617538e-07, 'epoch': 0.24437211825332247, 'num_input_tokens_seen': 3567255552, 'completed': '46.14% (1_701 / 3_687)', 'remaining time': '17:39:46', 'throughput': '9553.60', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 23:29:10,596 >> {'loss': 0.4474, 'grad_norm': 16.736602783203125, 'learning_rate': 6.437708457871455e-07, 'epoch': 0.24464334147002983, 'num_input_tokens_seen': 3569352704, 'completed': '46.16% (1_702 / 3_687)', 'remaining time': '17:39:10', 'throughput': '8686.12', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 23:29:45,133 >> {'loss': 0.5609, 'grad_norm': 17.7639217376709, 'learning_rate': 6.433759814935447e-07, 'epoch': 0.2449145646867372, 'num_input_tokens_seen': 3571449856, 'completed': '46.19% (1_703 / 3_687)', 'remaining time': '17:38:44', 'throughput': '7590.23', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 23:30:18,842 >> {'loss': 0.3755, 'grad_norm': 12.938990592956543, 'learning_rate': 6.429810420545072e-07, 'epoch': 0.24518578790344453, 'num_input_tokens_seen': 3573547008, 'completed': '46.22% (1_704 / 3_687)', 'remaining time': '17:38:15', 'throughput': '7776.77', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 23:30:49,195 >> {'loss': 0.6132, 'grad_norm': 19.107515335083008, 'learning_rate': 6.425860277878651e-07, 'epoch': 0.2454570111201519, 'num_input_tokens_seen': 3575644160, 'completed': '46.24% (1_705 / 3_687)', 'remaining time': '17:37:40', 'throughput': '8636.36', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 23:31:22,096 >> {'loss': 0.4449, 'grad_norm': 16.069114685058594, 'learning_rate': 6.42190939011511e-07, 'epoch': 0.24572823433685922, 'num_input_tokens_seen': 3577741312, 'completed': '46.27% (1_706 / 3_687)', 'remaining time': '17:37:10', 'throughput': '7967.70', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 23:31:55,142 >> {'loss': 0.4561, 'grad_norm': 18.453479766845703, 'learning_rate': 6.417957760433974e-07, 'epoch': 0.2459994575535666, 'num_input_tokens_seen': 3579838464, 'completed': '46.30% (1_707 / 3_687)', 'remaining time': '17:36:40', 'throughput': '7932.71', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 23:32:27,163 >> {'loss': 0.4495, 'grad_norm': 22.410266876220703, 'learning_rate': 6.414005392015364e-07, 'epoch': 0.24627068077027395, 'num_input_tokens_seen': 3581935616, 'completed': '46.32% (1_708 / 3_687)', 'remaining time': '17:36:08', 'throughput': '8186.51', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 23:33:00,223 >> {'loss': 0.647, 'grad_norm': 18.5291690826416, 'learning_rate': 6.410052288039994e-07, 'epoch': 0.24654190398698128, 'num_input_tokens_seen': 3584032768, 'completed': '46.35% (1_709 / 3_687)', 'remaining time': '17:35:38', 'throughput': '7929.38', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 23:33:30,498 >> {'loss': 0.3701, 'grad_norm': 15.33918571472168, 'learning_rate': 6.406098451689171e-07, 'epoch': 0.24681312720368864, 'num_input_tokens_seen': 3586129920, 'completed': '46.38% (1_710 / 3_687)', 'remaining time': '17:35:02', 'throughput': '8658.81', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 23:34:00,436 >> {'loss': 0.4231, 'grad_norm': 16.662532806396484, 'learning_rate': 6.402143886144797e-07, 'epoch': 0.24708435042039598, 'num_input_tokens_seen': 3588227072, 'completed': '46.41% (1_711 / 3_687)', 'remaining time': '17:34:26', 'throughput': '8756.41', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 23:34:30,781 >> {'loss': 0.431, 'grad_norm': 16.196252822875977, 'learning_rate': 6.39818859458935e-07, 'epoch': 0.24735557363710334, 'num_input_tokens_seen': 3590324224, 'completed': '46.43% (1_712 / 3_687)', 'remaining time': '17:33:50', 'throughput': '8638.79', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 23:34:58,869 >> {'loss': 0.9444, 'grad_norm': 24.346651077270508, 'learning_rate': 6.394232580205903e-07, 'epoch': 0.2476267968538107, 'num_input_tokens_seen': 3592421376, 'completed': '46.46% (1_713 / 3_687)', 'remaining time': '17:33:10', 'throughput': '9332.88', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 23:35:28,688 >> {'loss': 0.6498, 'grad_norm': 18.19024658203125, 'learning_rate': 6.390275846178102e-07, 'epoch': 0.24789802007051803, 'num_input_tokens_seen': 3594518528, 'completed': '46.49% (1_714 / 3_687)', 'remaining time': '17:32:33', 'throughput': '8791.09', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 23:35:58,894 >> {'loss': 0.279, 'grad_norm': 11.127647399902344, 'learning_rate': 6.386318395690178e-07, 'epoch': 0.2481692432872254, 'num_input_tokens_seen': 3596615680, 'completed': '46.51% (1_715 / 3_687)', 'remaining time': '17:31:57', 'throughput': '8678.65', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 23:36:29,936 >> {'loss': 0.5013, 'grad_norm': 21.56477165222168, 'learning_rate': 6.382360231926935e-07, 'epoch': 0.24844046650393273, 'num_input_tokens_seen': 3598712832, 'completed': '46.54% (1_716 / 3_687)', 'remaining time': '17:31:23', 'throughput': '8444.65', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 23:36:59,771 >> {'loss': 0.469, 'grad_norm': 15.533906936645508, 'learning_rate': 6.378401358073754e-07, 'epoch': 0.2487116897206401, 'num_input_tokens_seen': 3600809984, 'completed': '46.57% (1_717 / 3_687)', 'remaining time': '17:30:46', 'throughput': '8786.59', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 23:37:29,273 >> {'loss': 0.5628, 'grad_norm': 16.186729431152344, 'learning_rate': 6.374441777316587e-07, 'epoch': 0.24898291293734745, 'num_input_tokens_seen': 3602907136, 'completed': '46.60% (1_718 / 3_687)', 'remaining time': '17:30:09', 'throughput': '8885.68', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 23:37:58,559 >> {'loss': 0.4068, 'grad_norm': 14.087782859802246, 'learning_rate': 6.370481492841952e-07, 'epoch': 0.24925413615405478, 'num_input_tokens_seen': 3605004288, 'completed': '46.62% (1_719 / 3_687)', 'remaining time': '17:29:31', 'throughput': '8950.98', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 23:38:29,557 >> {'loss': 0.4257, 'grad_norm': 14.505183219909668, 'learning_rate': 6.366520507836934e-07, 'epoch': 0.24952535937076215, 'num_input_tokens_seen': 3607101440, 'completed': '46.65% (1_720 / 3_687)', 'remaining time': '17:28:57', 'throughput': '8457.01', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 23:38:59,510 >> {'loss': 0.3571, 'grad_norm': 14.800230979919434, 'learning_rate': 6.362558825489187e-07, 'epoch': 0.24979658258746948, 'num_input_tokens_seen': 3609198592, 'completed': '46.68% (1_721 / 3_687)', 'remaining time': '17:28:21', 'throughput': '8751.73', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 23:39:32,108 >> {'loss': 0.5663, 'grad_norm': 18.357351303100586, 'learning_rate': 6.35859644898692e-07, 'epoch': 0.2500678058041768, 'num_input_tokens_seen': 3611295744, 'completed': '46.70% (1_722 / 3_687)', 'remaining time': '17:27:50', 'throughput': '8041.62', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 23:40:00,610 >> {'loss': 0.6229, 'grad_norm': 18.787809371948242, 'learning_rate': 6.354633381518901e-07, 'epoch': 0.2503390290208842, 'num_input_tokens_seen': 3613392896, 'completed': '46.73% (1_723 / 3_687)', 'remaining time': '17:27:10', 'throughput': '9197.63', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 23:40:32,337 >> {'loss': 0.4966, 'grad_norm': 22.12680435180664, 'learning_rate': 6.350669626274461e-07, 'epoch': 0.25061025223759154, 'num_input_tokens_seen': 3615490048, 'completed': '46.76% (1_724 / 3_687)', 'remaining time': '17:26:38', 'throughput': '8262.44', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 23:41:03,315 >> {'loss': 0.4202, 'grad_norm': 14.827641487121582, 'learning_rate': 6.346705186443474e-07, 'epoch': 0.2508814754542989, 'num_input_tokens_seen': 3617587200, 'completed': '46.79% (1_725 / 3_687)', 'remaining time': '17:26:04', 'throughput': '8462.33', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 23:41:32,671 >> {'loss': 0.5388, 'grad_norm': 16.624746322631836, 'learning_rate': 6.342740065216371e-07, 'epoch': 0.25115269867100626, 'num_input_tokens_seen': 3619684352, 'completed': '46.81% (1_726 / 3_687)', 'remaining time': '17:25:26', 'throughput': '8929.72', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 23:42:01,681 >> {'loss': 0.3481, 'grad_norm': 13.560758590698242, 'learning_rate': 6.338774265784134e-07, 'epoch': 0.25142392188771356, 'num_input_tokens_seen': 3621781504, 'completed': '46.84% (1_727 / 3_687)', 'remaining time': '17:24:48', 'throughput': '9036.44', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 23:42:33,695 >> {'loss': 0.4449, 'grad_norm': 17.08588409423828, 'learning_rate': 6.334807791338286e-07, 'epoch': 0.2516951451044209, 'num_input_tokens_seen': 3623878656, 'completed': '46.87% (1_728 / 3_687)', 'remaining time': '17:24:16', 'throughput': '8188.32', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 23:43:04,242 >> {'loss': 0.6385, 'grad_norm': 25.524866104125977, 'learning_rate': 6.330840645070894e-07, 'epoch': 0.2519663683211283, 'num_input_tokens_seen': 3625975808, 'completed': '46.89% (1_729 / 3_687)', 'remaining time': '17:23:41', 'throughput': '8581.73', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 23:43:33,864 >> {'loss': 0.5051, 'grad_norm': 16.190298080444336, 'learning_rate': 6.326872830174566e-07, 'epoch': 0.25223759153783565, 'num_input_tokens_seen': 3628072960, 'completed': '46.92% (1_730 / 3_687)', 'remaining time': '17:23:04', 'throughput': '8849.49', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 23:44:04,127 >> {'loss': 0.4167, 'grad_norm': 17.590593338012695, 'learning_rate': 6.32290434984245e-07, 'epoch': 0.252508814754543, 'num_input_tokens_seen': 3630170112, 'completed': '46.95% (1_731 / 3_687)', 'remaining time': '17:22:28', 'throughput': '8662.23', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 23:44:36,499 >> {'loss': 0.5106, 'grad_norm': 23.287769317626953, 'learning_rate': 6.318935207268227e-07, 'epoch': 0.2527800379712503, 'num_input_tokens_seen': 3632267264, 'completed': '46.98% (1_732 / 3_687)', 'remaining time': '17:21:57', 'throughput': '8097.79', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 23:45:09,353 >> {'loss': 0.5227, 'grad_norm': 17.762920379638672, 'learning_rate': 6.314965405646113e-07, 'epoch': 0.2530512611879577, 'num_input_tokens_seen': 3634364416, 'completed': '47.00% (1_733 / 3_687)', 'remaining time': '17:21:27', 'throughput': '7979.23', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 23:45:39,052 >> {'loss': 0.7039, 'grad_norm': 23.90679931640625, 'learning_rate': 6.31099494817085e-07, 'epoch': 0.25332248440466504, 'num_input_tokens_seen': 3636461568, 'completed': '47.03% (1_734 / 3_687)', 'remaining time': '17:20:50', 'throughput': '8826.52', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 23:46:11,454 >> {'loss': 0.4308, 'grad_norm': 14.712848663330078, 'learning_rate': 6.307023838037714e-07, 'epoch': 0.2535937076213724, 'num_input_tokens_seen': 3638558720, 'completed': '47.06% (1_735 / 3_687)', 'remaining time': '17:20:19', 'throughput': '8090.30', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 23:46:44,369 >> {'loss': 0.6353, 'grad_norm': 20.47460174560547, 'learning_rate': 6.303052078442503e-07, 'epoch': 0.25386493083807976, 'num_input_tokens_seen': 3640655872, 'completed': '47.08% (1_736 / 3_687)', 'remaining time': '17:19:49', 'throughput': '7964.34', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 23:47:11,646 >> {'loss': 0.4013, 'grad_norm': 15.011195182800293, 'learning_rate': 6.299079672581537e-07, 'epoch': 0.25413615405478707, 'num_input_tokens_seen': 3642753024, 'completed': '47.11% (1_737 / 3_687)', 'remaining time': '17:19:07', 'throughput': '9610.47', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 23:47:41,924 >> {'loss': 0.5446, 'grad_norm': 17.400686264038086, 'learning_rate': 6.295106623651661e-07, 'epoch': 0.25440737727149443, 'num_input_tokens_seen': 3644850176, 'completed': '47.14% (1_738 / 3_687)', 'remaining time': '17:18:32', 'throughput': '8657.91', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 23:48:13,647 >> {'loss': 0.2983, 'grad_norm': 11.729777336120605, 'learning_rate': 6.291132934850225e-07, 'epoch': 0.2546786004882018, 'num_input_tokens_seen': 3646947328, 'completed': '47.17% (1_739 / 3_687)', 'remaining time': '17:18:00', 'throughput': '8263.59', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 23:48:45,326 >> {'loss': 0.8498, 'grad_norm': 21.60021209716797, 'learning_rate': 6.28715860937511e-07, 'epoch': 0.25494982370490915, 'num_input_tokens_seen': 3649044480, 'completed': '47.19% (1_740 / 3_687)', 'remaining time': '17:17:27', 'throughput': '8275.10', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 23:49:15,258 >> {'loss': 0.5259, 'grad_norm': 18.79129981994629, 'learning_rate': 6.283183650424701e-07, 'epoch': 0.2552210469216165, 'num_input_tokens_seen': 3651141632, 'completed': '47.22% (1_741 / 3_687)', 'remaining time': '17:16:51', 'throughput': '8757.73', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 23:49:45,104 >> {'loss': 0.6427, 'grad_norm': 19.276283264160156, 'learning_rate': 6.279208061197892e-07, 'epoch': 0.2554922701383238, 'num_input_tokens_seen': 3653238784, 'completed': '47.25% (1_742 / 3_687)', 'remaining time': '17:16:14', 'throughput': '8783.47', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 23:50:17,165 >> {'loss': 0.3971, 'grad_norm': 13.992378234863281, 'learning_rate': 6.275231844894086e-07, 'epoch': 0.2557634933550312, 'num_input_tokens_seen': 3655335936, 'completed': '47.27% (1_743 / 3_687)', 'remaining time': '17:15:43', 'throughput': '8176.42', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 23:50:47,591 >> {'loss': 0.4849, 'grad_norm': 16.478370666503906, 'learning_rate': 6.271255004713192e-07, 'epoch': 0.25603471657173854, 'num_input_tokens_seen': 3657433088, 'completed': '47.30% (1_744 / 3_687)', 'remaining time': '17:15:07', 'throughput': '8615.67', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 23:51:18,660 >> {'loss': 0.3075, 'grad_norm': 11.54108715057373, 'learning_rate': 6.267277543855618e-07, 'epoch': 0.2563059397884459, 'num_input_tokens_seen': 3659530240, 'completed': '47.33% (1_745 / 3_687)', 'remaining time': '17:14:34', 'throughput': '8437.54', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 23:51:47,469 >> {'loss': 0.5958, 'grad_norm': 19.243192672729492, 'learning_rate': 6.263299465522274e-07, 'epoch': 0.25657716300515326, 'num_input_tokens_seen': 3661627392, 'completed': '47.36% (1_746 / 3_687)', 'remaining time': '17:13:55', 'throughput': '9099.28', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 23:52:19,208 >> {'loss': 0.4542, 'grad_norm': 14.843926429748535, 'learning_rate': 6.259320772914566e-07, 'epoch': 0.25684838622186057, 'num_input_tokens_seen': 3663724544, 'completed': '47.38% (1_747 / 3_687)', 'remaining time': '17:13:23', 'throughput': '8259.50', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 23:52:49,908 >> {'loss': 0.8589, 'grad_norm': 22.970197677612305, 'learning_rate': 6.255341469234393e-07, 'epoch': 0.25711960943856793, 'num_input_tokens_seen': 3665821696, 'completed': '47.41% (1_748 / 3_687)', 'remaining time': '17:12:48', 'throughput': '8538.81', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 23:53:18,859 >> {'loss': 0.5667, 'grad_norm': 16.67009162902832, 'learning_rate': 6.25136155768415e-07, 'epoch': 0.2573908326552753, 'num_input_tokens_seen': 3667918848, 'completed': '47.44% (1_749 / 3_687)', 'remaining time': '17:12:10', 'throughput': '9054.59', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 23:53:53,336 >> {'loss': 0.4136, 'grad_norm': 13.778572082519531, 'learning_rate': 6.247381041466716e-07, 'epoch': 0.25766205587198265, 'num_input_tokens_seen': 3670016000, 'completed': '47.46% (1_750 / 3_687)', 'remaining time': '17:11:43', 'throughput': '7603.48', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 23:54:25,667 >> {'loss': 0.3029, 'grad_norm': 11.786734580993652, 'learning_rate': 6.243399923785459e-07, 'epoch': 0.25793327908869, 'num_input_tokens_seen': 3672113152, 'completed': '47.49% (1_751 / 3_687)', 'remaining time': '17:11:12', 'throughput': '8108.26', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 23:54:55,262 >> {'loss': 0.614, 'grad_norm': 18.904006958007812, 'learning_rate': 6.239418207844232e-07, 'epoch': 0.2582045023053973, 'num_input_tokens_seen': 3674210304, 'completed': '47.52% (1_752 / 3_687)', 'remaining time': '17:10:35', 'throughput': '8857.64', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 23:55:27,206 >> {'loss': 0.4589, 'grad_norm': 16.964792251586914, 'learning_rate': 6.23543589684737e-07, 'epoch': 0.2584757255221047, 'num_input_tokens_seen': 3676307456, 'completed': '47.55% (1_753 / 3_687)', 'remaining time': '17:10:03', 'throughput': '8206.22', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 23:55:58,670 >> {'loss': 0.4943, 'grad_norm': 15.05766487121582, 'learning_rate': 6.231452993999683e-07, 'epoch': 0.25874694873881204, 'num_input_tokens_seen': 3678404608, 'completed': '47.57% (1_754 / 3_687)', 'remaining time': '17:09:30', 'throughput': '8331.77', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 23:56:27,761 >> {'loss': 0.558, 'grad_norm': 18.096086502075195, 'learning_rate': 6.22746950250646e-07, 'epoch': 0.2590181719555194, 'num_input_tokens_seen': 3680501760, 'completed': '47.60% (1_755 / 3_687)', 'remaining time': '17:08:53', 'throughput': '9011.19', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 23:56:57,402 >> {'loss': 0.4175, 'grad_norm': 15.403691291809082, 'learning_rate': 6.223485425573463e-07, 'epoch': 0.25928939517222677, 'num_input_tokens_seen': 3682598912, 'completed': '47.63% (1_756 / 3_687)', 'remaining time': '17:08:16', 'throughput': '8843.74', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 23:57:29,116 >> {'loss': 0.3693, 'grad_norm': 14.380963325500488, 'learning_rate': 6.219500766406926e-07, 'epoch': 0.2595606183889341, 'num_input_tokens_seen': 3684696064, 'completed': '47.65% (1_757 / 3_687)', 'remaining time': '17:07:44', 'throughput': '8266.06', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 23:58:04,841 >> {'loss': 0.5975, 'grad_norm': 20.143505096435547, 'learning_rate': 6.215515528213553e-07, 'epoch': 0.25983184160564143, 'num_input_tokens_seen': 3686793216, 'completed': '47.68% (1_758 / 3_687)', 'remaining time': '17:07:19', 'throughput': '7337.83', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 23:58:34,241 >> {'loss': 0.3873, 'grad_norm': 18.09722900390625, 'learning_rate': 6.211529714200509e-07, 'epoch': 0.2601030648223488, 'num_input_tokens_seen': 3688890368, 'completed': '47.71% (1_759 / 3_687)', 'remaining time': '17:06:42', 'throughput': '8916.29', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 23:59:05,361 >> {'loss': 0.511, 'grad_norm': 18.81859588623047, 'learning_rate': 6.207543327575426e-07, 'epoch': 0.26037428803905616, 'num_input_tokens_seen': 3690987520, 'completed': '47.74% (1_760 / 3_687)', 'remaining time': '17:06:09', 'throughput': '8423.76', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-25 23:59:37,070 >> {'loss': 0.5123, 'grad_norm': 18.815465927124023, 'learning_rate': 6.203556371546399e-07, 'epoch': 0.2606455112557635, 'num_input_tokens_seen': 3693084672, 'completed': '47.76% (1_761 / 3_687)', 'remaining time': '17:05:36', 'throughput': '8267.25', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 00:00:06,286 >> {'loss': 0.4696, 'grad_norm': 15.527567863464355, 'learning_rate': 6.199568849321975e-07, 'epoch': 0.2609167344724708, 'num_input_tokens_seen': 3695181824, 'completed': '47.79% (1_762 / 3_687)', 'remaining time': '17:04:59', 'throughput': '8972.32', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 00:00:38,029 >> {'loss': 0.4165, 'grad_norm': 15.812019348144531, 'learning_rate': 6.195580764111163e-07, 'epoch': 0.2611879576891782, 'num_input_tokens_seen': 3697278976, 'completed': '47.82% (1_763 / 3_687)', 'remaining time': '17:04:26', 'throughput': '8258.37', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 00:01:09,462 >> {'loss': 0.2559, 'grad_norm': 11.180591583251953, 'learning_rate': 6.191592119123419e-07, 'epoch': 0.26145918090588555, 'num_input_tokens_seen': 3699376128, 'completed': '47.84% (1_764 / 3_687)', 'remaining time': '17:03:53', 'throughput': '8339.90', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 00:01:38,798 >> {'loss': 0.1839, 'grad_norm': 8.583144187927246, 'learning_rate': 6.187602917568655e-07, 'epoch': 0.2617304041225929, 'num_input_tokens_seen': 3701473280, 'completed': '47.87% (1_765 / 3_687)', 'remaining time': '17:03:16', 'throughput': '8935.94', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 00:02:11,682 >> {'loss': 0.2641, 'grad_norm': 12.009425163269043, 'learning_rate': 6.183613162657229e-07, 'epoch': 0.26200162733930027, 'num_input_tokens_seen': 3703570432, 'completed': '47.90% (1_766 / 3_687)', 'remaining time': '17:02:46', 'throughput': '7971.94', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 00:02:43,684 >> {'loss': 0.4814, 'grad_norm': 14.513581275939941, 'learning_rate': 6.179622857599942e-07, 'epoch': 0.2622728505560076, 'num_input_tokens_seen': 3705667584, 'completed': '47.93% (1_767 / 3_687)', 'remaining time': '17:02:14', 'throughput': '8191.20', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 00:03:16,289 >> {'loss': 0.6047, 'grad_norm': 23.496122360229492, 'learning_rate': 6.175632005608043e-07, 'epoch': 0.26254407377271494, 'num_input_tokens_seen': 3707764736, 'completed': '47.95% (1_768 / 3_687)', 'remaining time': '17:01:44', 'throughput': '8039.94', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 00:03:48,135 >> {'loss': 0.4534, 'grad_norm': 19.470165252685547, 'learning_rate': 6.171640609893213e-07, 'epoch': 0.2628152969894223, 'num_input_tokens_seen': 3709861888, 'completed': '47.98% (1_769 / 3_687)', 'remaining time': '17:01:12', 'throughput': '8231.70', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 00:04:17,395 >> {'loss': 0.403, 'grad_norm': 15.71687126159668, 'learning_rate': 6.167648673667581e-07, 'epoch': 0.26308652020612966, 'num_input_tokens_seen': 3711959040, 'completed': '48.01% (1_770 / 3_687)', 'remaining time': '17:00:34', 'throughput': '8959.06', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 00:04:50,064 >> {'loss': 0.6622, 'grad_norm': 23.09616470336914, 'learning_rate': 6.1636562001437e-07, 'epoch': 0.263357743422837, 'num_input_tokens_seen': 3714056192, 'completed': '48.03% (1_771 / 3_687)', 'remaining time': '17:00:04', 'throughput': '8024.23', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 00:05:20,743 >> {'loss': 0.4457, 'grad_norm': 15.076805114746094, 'learning_rate': 6.159663192534561e-07, 'epoch': 0.2636289666395443, 'num_input_tokens_seen': 3716153344, 'completed': '48.06% (1_772 / 3_687)', 'remaining time': '16:59:29', 'throughput': '8544.81', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 00:05:49,741 >> {'loss': 0.3627, 'grad_norm': 13.646811485290527, 'learning_rate': 6.155669654053592e-07, 'epoch': 0.2639001898562517, 'num_input_tokens_seen': 3718250496, 'completed': '48.09% (1_773 / 3_687)', 'remaining time': '16:58:52', 'throughput': '9040.17', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 00:06:19,625 >> {'loss': 0.6803, 'grad_norm': 20.863006591796875, 'learning_rate': 6.151675587914631e-07, 'epoch': 0.26417141307295905, 'num_input_tokens_seen': 3720347648, 'completed': '48.11% (1_774 / 3_687)', 'remaining time': '16:58:16', 'throughput': '8772.00', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 00:06:46,039 >> {'loss': 0.7848, 'grad_norm': 22.40181541442871, 'learning_rate': 6.147680997331958e-07, 'epoch': 0.2644426362896664, 'num_input_tokens_seen': 3722444800, 'completed': '48.14% (1_775 / 3_687)', 'remaining time': '16:57:33', 'throughput': '9924.52', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 00:07:19,456 >> {'loss': 0.5724, 'grad_norm': 20.66728973388672, 'learning_rate': 6.143685885520263e-07, 'epoch': 0.26471385950637377, 'num_input_tokens_seen': 3724541952, 'completed': '48.17% (1_776 / 3_687)', 'remaining time': '16:57:04', 'throughput': '7844.44', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 00:07:51,954 >> {'loss': 0.3131, 'grad_norm': 16.2211971282959, 'learning_rate': 6.13969025569466e-07, 'epoch': 0.2649850827230811, 'num_input_tokens_seen': 3726639104, 'completed': '48.20% (1_777 / 3_687)', 'remaining time': '16:56:33', 'throughput': '8066.61', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 00:08:22,035 >> {'loss': 0.3258, 'grad_norm': 14.444147109985352, 'learning_rate': 6.135694111070684e-07, 'epoch': 0.26525630593978844, 'num_input_tokens_seen': 3728736256, 'completed': '48.22% (1_778 / 3_687)', 'remaining time': '16:55:57', 'throughput': '8714.65', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 00:08:52,012 >> {'loss': 0.4398, 'grad_norm': 15.806771278381348, 'learning_rate': 6.131697454864277e-07, 'epoch': 0.2655275291564958, 'num_input_tokens_seen': 3730833408, 'completed': '48.25% (1_779 / 3_687)', 'remaining time': '16:55:22', 'throughput': '8744.82', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 00:09:22,046 >> {'loss': 0.6636, 'grad_norm': 19.829689025878906, 'learning_rate': 6.127700290291794e-07, 'epoch': 0.26579875237320316, 'num_input_tokens_seen': 3732930560, 'completed': '48.28% (1_780 / 3_687)', 'remaining time': '16:54:46', 'throughput': '8728.06', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 00:09:52,042 >> {'loss': 0.4753, 'grad_norm': 16.643007278442383, 'learning_rate': 6.123702620570005e-07, 'epoch': 0.2660699755899105, 'num_input_tokens_seen': 3735027712, 'completed': '48.30% (1_781 / 3_687)', 'remaining time': '16:54:10', 'throughput': '8739.46', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 00:10:24,794 >> {'loss': 0.4912, 'grad_norm': 21.29400062561035, 'learning_rate': 6.11970444891608e-07, 'epoch': 0.26634119880661783, 'num_input_tokens_seen': 3737124864, 'completed': '48.33% (1_782 / 3_687)', 'remaining time': '16:53:40', 'throughput': '8003.84', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 00:10:54,068 >> {'loss': 0.2683, 'grad_norm': 14.493866920471191, 'learning_rate': 6.115705778547597e-07, 'epoch': 0.2666124220233252, 'num_input_tokens_seen': 3739222016, 'completed': '48.36% (1_783 / 3_687)', 'remaining time': '16:53:03', 'throughput': '8954.75', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 00:11:23,248 >> {'loss': 0.6509, 'grad_norm': 19.871868133544922, 'learning_rate': 6.111706612682532e-07, 'epoch': 0.26688364524003255, 'num_input_tokens_seen': 3741319168, 'completed': '48.39% (1_784 / 3_687)', 'remaining time': '16:52:26', 'throughput': '8983.80', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 00:11:53,468 >> {'loss': 0.3895, 'grad_norm': 15.187943458557129, 'learning_rate': 6.107706954539261e-07, 'epoch': 0.2671548684567399, 'num_input_tokens_seen': 3743416320, 'completed': '48.41% (1_785 / 3_687)', 'remaining time': '16:51:51', 'throughput': '8674.43', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 00:12:24,128 >> {'loss': 0.54, 'grad_norm': 15.838203430175781, 'learning_rate': 6.103706807336559e-07, 'epoch': 0.2674260916734473, 'num_input_tokens_seen': 3745513472, 'completed': '48.44% (1_786 / 3_687)', 'remaining time': '16:51:16', 'throughput': '8550.06', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 00:12:56,043 >> {'loss': 0.4153, 'grad_norm': 12.953702926635742, 'learning_rate': 6.099706174293592e-07, 'epoch': 0.2676973148901546, 'num_input_tokens_seen': 3747610624, 'completed': '48.47% (1_787 / 3_687)', 'remaining time': '16:50:44', 'throughput': '8213.78', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 00:13:26,081 >> {'loss': 0.2817, 'grad_norm': 14.383577346801758, 'learning_rate': 6.095705058629914e-07, 'epoch': 0.26796853810686194, 'num_input_tokens_seen': 3749707776, 'completed': '48.49% (1_788 / 3_687)', 'remaining time': '16:50:09', 'throughput': '8727.18', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 00:13:54,960 >> {'loss': 0.3399, 'grad_norm': 14.063627243041992, 'learning_rate': 6.091703463565475e-07, 'epoch': 0.2682397613235693, 'num_input_tokens_seen': 3751804928, 'completed': '48.52% (1_789 / 3_687)', 'remaining time': '16:49:31', 'throughput': '9077.22', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 00:14:26,084 >> {'loss': 0.5057, 'grad_norm': 16.237741470336914, 'learning_rate': 6.087701392320606e-07, 'epoch': 0.26851098454027666, 'num_input_tokens_seen': 3753902080, 'completed': '48.55% (1_790 / 3_687)', 'remaining time': '16:48:58', 'throughput': '8422.72', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 00:14:56,498 >> {'loss': 0.3234, 'grad_norm': 12.57223129272461, 'learning_rate': 6.083698848116018e-07, 'epoch': 0.26878220775698397, 'num_input_tokens_seen': 3755999232, 'completed': '48.58% (1_791 / 3_687)', 'remaining time': '16:48:23', 'throughput': '8619.05', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 00:15:20,739 >> {'loss': 0.9309, 'grad_norm': 26.410602569580078, 'learning_rate': 6.079695834172808e-07, 'epoch': 0.26905343097369133, 'num_input_tokens_seen': 3758096384, 'completed': '48.60% (1_792 / 3_687)', 'remaining time': '16:47:36', 'throughput': '10814.24', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 00:15:54,764 >> {'loss': 0.4636, 'grad_norm': 17.888185501098633, 'learning_rate': 6.075692353712451e-07, 'epoch': 0.2693246541903987, 'num_input_tokens_seen': 3760193536, 'completed': '48.63% (1_793 / 3_687)', 'remaining time': '16:47:08', 'throughput': '7704.57', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 00:16:23,571 >> {'loss': 0.5735, 'grad_norm': 20.749839782714844, 'learning_rate': 6.071688409956793e-07, 'epoch': 0.26959587740710605, 'num_input_tokens_seen': 3762290688, 'completed': '48.66% (1_794 / 3_687)', 'remaining time': '16:46:31', 'throughput': '9099.97', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 00:16:51,933 >> {'loss': 0.5885, 'grad_norm': 23.78219223022461, 'learning_rate': 6.06768400612806e-07, 'epoch': 0.2698671006238134, 'num_input_tokens_seen': 3764387840, 'completed': '48.68% (1_795 / 3_687)', 'remaining time': '16:45:52', 'throughput': '9242.55', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 00:17:20,253 >> {'loss': 0.7039, 'grad_norm': 25.068920135498047, 'learning_rate': 6.063679145448838e-07, 'epoch': 0.2701383238405207, 'num_input_tokens_seen': 3766484992, 'completed': '48.71% (1_796 / 3_687)', 'remaining time': '16:45:13', 'throughput': '9256.71', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 00:17:53,128 >> {'loss': 0.3652, 'grad_norm': 15.582983016967773, 'learning_rate': 6.05967383114209e-07, 'epoch': 0.2704095470572281, 'num_input_tokens_seen': 3768582144, 'completed': '48.74% (1_797 / 3_687)', 'remaining time': '16:44:43', 'throughput': '7973.93', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 00:18:24,129 >> {'loss': 0.6668, 'grad_norm': 27.859983444213867, 'learning_rate': 6.055668066431142e-07, 'epoch': 0.27068077027393544, 'num_input_tokens_seen': 3770679296, 'completed': '48.77% (1_798 / 3_687)', 'remaining time': '16:44:10', 'throughput': '8455.99', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 00:18:54,144 >> {'loss': 0.3411, 'grad_norm': 12.221877098083496, 'learning_rate': 6.051661854539677e-07, 'epoch': 0.2709519934906428, 'num_input_tokens_seen': 3772776448, 'completed': '48.79% (1_799 / 3_687)', 'remaining time': '16:43:34', 'throughput': '8733.80', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 00:19:24,205 >> {'loss': 0.794, 'grad_norm': 20.4038143157959, 'learning_rate': 6.047655198691742e-07, 'epoch': 0.27122321670735017, 'num_input_tokens_seen': 3774873600, 'completed': '48.82% (1_800 / 3_687)', 'remaining time': '16:42:59', 'throughput': '8720.45', 'gpu_mem_free': '5581MB'} +/scratch3/workspace/ctpham_umass_edu-ft/envs/prolong-final/lib/python3.10/site-packages/torch/distributed/fsdp/fully_sharded_data_parallel.py:689: FutureWarning: FSDP.state_dict_type() and FSDP.set_state_dict_type() are being deprecated. Please use APIs, get_state_dict() and set_state_dict(), which can support different parallelisms, FSDP1, FSDP2, DDP. API doc: https://pytorch.org/docs/stable/distributed.checkpoint.html#torch.distributed.checkpoint.state_dict.get_state_dict .Tutorial: https://pytorch.org/tutorials/recipes/distributed_checkpoint_recipe.html . + warnings.warn( +[INFO|trainer.py:3503] 2024-12-26 00:19:48,102 >> Saving model checkpoint to /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/checkpoint-1800 +[INFO|configuration_utils.py:472] 2024-12-26 00:19:48,105 >> Configuration saved in /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/checkpoint-1800/config.json +[INFO|configuration_utils.py:807] 2024-12-26 00:19:48,106 >> Configuration saved in /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/checkpoint-1800/generation_config.json +[INFO|modeling_utils.py:2807] 2024-12-26 00:20:43,418 >> The model is bigger than the maximum size per checkpoint (5GB) and is going to be split in 7 checkpoint shards. You can find where each parameters has been saved in the index located at /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/checkpoint-1800/model.safetensors.index.json. +[INFO|tokenization_utils_base.py:2684] 2024-12-26 00:20:43,422 >> tokenizer config file saved in /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/checkpoint-1800/tokenizer_config.json +[INFO|tokenization_utils_base.py:2693] 2024-12-26 00:20:43,423 >> Special tokens file saved in /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/checkpoint-1800/special_tokens_map.json +/scratch3/workspace/ctpham_umass_edu-ft/envs/prolong-final/lib/python3.10/site-packages/torch/distributed/fsdp/fully_sharded_data_parallel.py:689: FutureWarning: FSDP.state_dict_type() and FSDP.set_state_dict_type() are being deprecated. Please use APIs, get_state_dict() and set_state_dict(), which can support different parallelisms, FSDP1, FSDP2, DDP. API doc: https://pytorch.org/docs/stable/distributed.checkpoint.html#torch.distributed.checkpoint.state_dict.get_state_dict .Tutorial: https://pytorch.org/tutorials/recipes/distributed_checkpoint_recipe.html . + warnings.warn( +[WARNING|trainer.py:868] 2024-12-26 00:24:38,073 >> Save streaming dataset state: {'epoch': 0, 'sample_in_epoch': 3600, 'num_canonical_nodes': 1, 'shuffle_seed': 42, 'initial_physical_nodes': 1} +/scratch3/workspace/ctpham_umass_edu-ft/envs/prolong-final/lib/python3.10/site-packages/torch/utils/checkpoint.py:1399: FutureWarning: `torch.cpu.amp.autocast(args...)` is deprecated. Please use `torch.amp.autocast('cpu', args...)` instead. + with device_autocast_ctx, torch.cpu.amp.autocast(**cpu_autocast_kwargs), recompute_context: # type: ignore[attr-defined] +[INFO|trainer.py:175] 2024-12-26 00:25:09,242 >> {'loss': 0.2082, 'grad_norm': 13.717342376708984, 'learning_rate': 6.043648102111745e-07, 'epoch': 0.2714944399240575, 'num_input_tokens_seen': 3776970752, 'completed': '48.85% (1_801 / 3_687)', 'remaining time': '16:52:17', 'throughput': '759.76', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 00:25:39,665 >> {'loss': 0.6036, 'grad_norm': 19.058935165405273, 'learning_rate': 6.039640568024443e-07, 'epoch': 0.27176566314076483, 'num_input_tokens_seen': 3779067904, 'completed': '48.87% (1_802 / 3_687)', 'remaining time': '16:51:41', 'throughput': '8616.59', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 00:26:08,060 >> {'loss': 0.5712, 'grad_norm': 15.670707702636719, 'learning_rate': 6.035632599654946e-07, 'epoch': 0.2720368863574722, 'num_input_tokens_seen': 3781165056, 'completed': '48.90% (1_803 / 3_687)', 'remaining time': '16:51:02', 'throughput': '9231.82', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 00:26:38,625 >> {'loss': 0.3603, 'grad_norm': 13.817047119140625, 'learning_rate': 6.031624200228715e-07, 'epoch': 0.27230810957417956, 'num_input_tokens_seen': 3783262208, 'completed': '48.93% (1_804 / 3_687)', 'remaining time': '16:50:27', 'throughput': '8576.61', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 00:27:10,846 >> {'loss': 0.4361, 'grad_norm': 15.026437759399414, 'learning_rate': 6.027615372971558e-07, 'epoch': 0.2725793327908869, 'num_input_tokens_seen': 3785359360, 'completed': '48.96% (1_805 / 3_687)', 'remaining time': '16:49:55', 'throughput': '8136.03', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 00:27:40,910 >> {'loss': 0.4687, 'grad_norm': 16.934778213500977, 'learning_rate': 6.023606121109626e-07, 'epoch': 0.2728505560075942, 'num_input_tokens_seen': 3787456512, 'completed': '48.98% (1_806 / 3_687)', 'remaining time': '16:49:18', 'throughput': '8719.85', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 00:28:12,974 >> {'loss': 0.423, 'grad_norm': 18.090913772583008, 'learning_rate': 6.019596447869413e-07, 'epoch': 0.2731217792243016, 'num_input_tokens_seen': 3789553664, 'completed': '49.01% (1_807 / 3_687)', 'remaining time': '16:48:46', 'throughput': '8175.74', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 00:28:43,160 >> {'loss': 0.3783, 'grad_norm': 15.30374813079834, 'learning_rate': 6.015586356477749e-07, 'epoch': 0.27339300244100895, 'num_input_tokens_seen': 3791650816, 'completed': '49.04% (1_808 / 3_687)', 'remaining time': '16:48:10', 'throughput': '8683.60', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 00:29:14,167 >> {'loss': 0.576, 'grad_norm': 21.7882080078125, 'learning_rate': 6.011575850161805e-07, 'epoch': 0.2736642256577163, 'num_input_tokens_seen': 3793747968, 'completed': '49.06% (1_809 / 3_687)', 'remaining time': '16:47:36', 'throughput': '8454.34', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 00:29:41,556 >> {'loss': 0.5201, 'grad_norm': 17.875171661376953, 'learning_rate': 6.007564932149086e-07, 'epoch': 0.27393544887442367, 'num_input_tokens_seen': 3795845120, 'completed': '49.09% (1_810 / 3_687)', 'remaining time': '16:46:55', 'throughput': '9571.38', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 00:30:14,062 >> {'loss': 0.3927, 'grad_norm': 15.981590270996094, 'learning_rate': 6.003553605667423e-07, 'epoch': 0.274206672091131, 'num_input_tokens_seen': 3797942272, 'completed': '49.12% (1_811 / 3_687)', 'remaining time': '16:46:23', 'throughput': '8064.32', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 00:30:43,113 >> {'loss': 0.4337, 'grad_norm': 14.422945022583008, 'learning_rate': 5.999541873944979e-07, 'epoch': 0.27447789530783834, 'num_input_tokens_seen': 3800039424, 'completed': '49.15% (1_812 / 3_687)', 'remaining time': '16:45:45', 'throughput': '9023.85', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 00:31:15,424 >> {'loss': 0.4126, 'grad_norm': 17.687833786010742, 'learning_rate': 5.995529740210244e-07, 'epoch': 0.2747491185245457, 'num_input_tokens_seen': 3802136576, 'completed': '49.17% (1_813 / 3_687)', 'remaining time': '16:45:13', 'throughput': '8113.10', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 00:31:46,617 >> {'loss': 0.2607, 'grad_norm': 11.91661262512207, 'learning_rate': 5.99151720769203e-07, 'epoch': 0.27502034174125306, 'num_input_tokens_seen': 3804233728, 'completed': '49.20% (1_814 / 3_687)', 'remaining time': '16:44:39', 'throughput': '8403.94', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 00:32:14,528 >> {'loss': 0.4029, 'grad_norm': 13.01582145690918, 'learning_rate': 5.987504279619473e-07, 'epoch': 0.2752915649579604, 'num_input_tokens_seen': 3806330880, 'completed': '49.23% (1_815 / 3_687)', 'remaining time': '16:43:59', 'throughput': '9391.88', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 00:32:46,271 >> {'loss': 0.3179, 'grad_norm': 14.431076049804688, 'learning_rate': 5.98349095922202e-07, 'epoch': 0.2755627881746677, 'num_input_tokens_seen': 3808428032, 'completed': '49.25% (1_816 / 3_687)', 'remaining time': '16:43:26', 'throughput': '8258.35', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 00:33:15,598 >> {'loss': 0.9278, 'grad_norm': 23.85053825378418, 'learning_rate': 5.979477249729442e-07, 'epoch': 0.2758340113913751, 'num_input_tokens_seen': 3810525184, 'completed': '49.28% (1_817 / 3_687)', 'remaining time': '16:42:49', 'throughput': '8938.74', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 00:33:46,164 >> {'loss': 0.5163, 'grad_norm': 16.68154525756836, 'learning_rate': 5.975463154371822e-07, 'epoch': 0.27610523460808245, 'num_input_tokens_seen': 3812622336, 'completed': '49.31% (1_818 / 3_687)', 'remaining time': '16:42:13', 'throughput': '8576.24', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 00:34:15,642 >> {'loss': 0.4715, 'grad_norm': 20.487937927246094, 'learning_rate': 5.971448676379544e-07, 'epoch': 0.2763764578247898, 'num_input_tokens_seen': 3814719488, 'completed': '49.34% (1_819 / 3_687)', 'remaining time': '16:41:36', 'throughput': '8892.88', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 00:34:47,881 >> {'loss': 0.3283, 'grad_norm': 11.562193870544434, 'learning_rate': 5.967433818983311e-07, 'epoch': 0.2766476810414972, 'num_input_tokens_seen': 3816816640, 'completed': '49.36% (1_820 / 3_687)', 'remaining time': '16:41:04', 'throughput': '8131.27', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 00:35:17,497 >> {'loss': 0.3784, 'grad_norm': 14.949207305908203, 'learning_rate': 5.963418585414129e-07, 'epoch': 0.2769189042582045, 'num_input_tokens_seen': 3818913792, 'completed': '49.39% (1_821 / 3_687)', 'remaining time': '16:40:27', 'throughput': '8851.69', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 00:35:48,494 >> {'loss': 0.2296, 'grad_norm': 13.64500617980957, 'learning_rate': 5.959402978903306e-07, 'epoch': 0.27719012747491184, 'num_input_tokens_seen': 3821010944, 'completed': '49.42% (1_822 / 3_687)', 'remaining time': '16:39:53', 'throughput': '8456.91', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 00:36:20,347 >> {'loss': 0.7141, 'grad_norm': 20.391284942626953, 'learning_rate': 5.955387002682445e-07, 'epoch': 0.2774613506916192, 'num_input_tokens_seen': 3823108096, 'completed': '49.44% (1_823 / 3_687)', 'remaining time': '16:39:20', 'throughput': '8229.82', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 00:36:51,617 >> {'loss': 0.3659, 'grad_norm': 12.009023666381836, 'learning_rate': 5.951370659983452e-07, 'epoch': 0.27773257390832656, 'num_input_tokens_seen': 3825205248, 'completed': '49.47% (1_824 / 3_687)', 'remaining time': '16:38:47', 'throughput': '8383.26', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 00:37:21,537 >> {'loss': 0.273, 'grad_norm': 12.802525520324707, 'learning_rate': 5.94735395403853e-07, 'epoch': 0.2780037971250339, 'num_input_tokens_seen': 3827302400, 'completed': '49.50% (1_825 / 3_687)', 'remaining time': '16:38:10', 'throughput': '8761.51', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 00:37:50,520 >> {'loss': 0.7285, 'grad_norm': 22.112329483032227, 'learning_rate': 5.943336888080167e-07, 'epoch': 0.27827502034174123, 'num_input_tokens_seen': 3829399552, 'completed': '49.53% (1_826 / 3_687)', 'remaining time': '16:37:32', 'throughput': '9044.71', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 00:38:20,791 >> {'loss': 0.5338, 'grad_norm': 17.80236053466797, 'learning_rate': 5.939319465341148e-07, 'epoch': 0.2785462435584486, 'num_input_tokens_seen': 3831496704, 'completed': '49.55% (1_827 / 3_687)', 'remaining time': '16:36:57', 'throughput': '8660.04', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 00:38:51,950 >> {'loss': 0.444, 'grad_norm': 17.097667694091797, 'learning_rate': 5.93530168905454e-07, 'epoch': 0.27881746677515595, 'num_input_tokens_seen': 3833593856, 'completed': '49.58% (1_828 / 3_687)', 'remaining time': '16:36:23', 'throughput': '8413.11', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 00:39:23,073 >> {'loss': 0.4893, 'grad_norm': 16.976318359375, 'learning_rate': 5.931283562453696e-07, 'epoch': 0.2790886899918633, 'num_input_tokens_seen': 3835691008, 'completed': '49.61% (1_829 / 3_687)', 'remaining time': '16:35:49', 'throughput': '8422.76', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 00:39:50,033 >> {'loss': 0.4142, 'grad_norm': 14.774353981018066, 'learning_rate': 5.927265088772255e-07, 'epoch': 0.2793599132085707, 'num_input_tokens_seen': 3837788160, 'completed': '49.63% (1_830 / 3_687)', 'remaining time': '16:35:07', 'throughput': '9723.48', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 00:40:20,310 >> {'loss': 0.5785, 'grad_norm': 17.598939895629883, 'learning_rate': 5.923246271244127e-07, 'epoch': 0.279631136425278, 'num_input_tokens_seen': 3839885312, 'completed': '49.66% (1_831 / 3_687)', 'remaining time': '16:34:32', 'throughput': '8658.22', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 00:40:53,408 >> {'loss': 0.4257, 'grad_norm': 18.117116928100586, 'learning_rate': 5.919227113103508e-07, 'epoch': 0.27990235964198534, 'num_input_tokens_seen': 3841982464, 'completed': '49.69% (1_832 / 3_687)', 'remaining time': '16:34:01', 'throughput': '7920.29', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 00:41:22,719 >> {'loss': 0.279, 'grad_norm': 12.194579124450684, 'learning_rate': 5.915207617584858e-07, 'epoch': 0.2801735828586927, 'num_input_tokens_seen': 3844079616, 'completed': '49.72% (1_833 / 3_687)', 'remaining time': '16:33:24', 'throughput': '8943.53', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 00:41:53,504 >> {'loss': 0.4616, 'grad_norm': 15.989289283752441, 'learning_rate': 5.911187787922918e-07, 'epoch': 0.28044480607540007, 'num_input_tokens_seen': 3846176768, 'completed': '49.74% (1_834 / 3_687)', 'remaining time': '16:32:49', 'throughput': '8515.08', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 00:42:27,366 >> {'loss': 0.3184, 'grad_norm': 14.382404327392578, 'learning_rate': 5.907167627352697e-07, 'epoch': 0.2807160292921074, 'num_input_tokens_seen': 3848273920, 'completed': '49.77% (1_835 / 3_687)', 'remaining time': '16:32:20', 'throughput': '7741.57', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 00:42:58,160 >> {'loss': 0.4872, 'grad_norm': 15.597938537597656, 'learning_rate': 5.90314713910946e-07, 'epoch': 0.28098725250881473, 'num_input_tokens_seen': 3850371072, 'completed': '49.80% (1_836 / 3_687)', 'remaining time': '16:31:46', 'throughput': '8512.92', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 00:43:27,040 >> {'loss': 0.4539, 'grad_norm': 14.058372497558594, 'learning_rate': 5.899126326428746e-07, 'epoch': 0.2812584757255221, 'num_input_tokens_seen': 3852468224, 'completed': '49.82% (1_837 / 3_687)', 'remaining time': '16:31:08', 'throughput': '9077.10', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 00:43:56,305 >> {'loss': 0.742, 'grad_norm': 23.597742080688477, 'learning_rate': 5.895105192546353e-07, 'epoch': 0.28152969894222946, 'num_input_tokens_seen': 3854565376, 'completed': '49.85% (1_838 / 3_687)', 'remaining time': '16:30:31', 'throughput': '8957.59', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 00:44:26,254 >> {'loss': 0.243, 'grad_norm': 10.927760124206543, 'learning_rate': 5.891083740698337e-07, 'epoch': 0.2818009221589368, 'num_input_tokens_seen': 3856662528, 'completed': '49.88% (1_839 / 3_687)', 'remaining time': '16:29:55', 'throughput': '8752.93', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 00:44:59,172 >> {'loss': 0.4831, 'grad_norm': 19.77141571044922, 'learning_rate': 5.887061974121007e-07, 'epoch': 0.2820721453756442, 'num_input_tokens_seen': 3858759680, 'completed': '49.91% (1_840 / 3_687)', 'remaining time': '16:29:24', 'throughput': '7963.42', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 00:45:29,194 >> {'loss': 0.6657, 'grad_norm': 18.980165481567383, 'learning_rate': 5.883039896050928e-07, 'epoch': 0.2823433685923515, 'num_input_tokens_seen': 3860856832, 'completed': '49.93% (1_841 / 3_687)', 'remaining time': '16:28:48', 'throughput': '8731.72', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 00:46:00,539 >> {'loss': 0.4045, 'grad_norm': 14.606121063232422, 'learning_rate': 5.879017509724919e-07, 'epoch': 0.28261459180905885, 'num_input_tokens_seen': 3862953984, 'completed': '49.96% (1_842 / 3_687)', 'remaining time': '16:28:14', 'throughput': '8363.28', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 00:46:32,693 >> {'loss': 0.5391, 'grad_norm': 18.364437103271484, 'learning_rate': 5.874994818380039e-07, 'epoch': 0.2828858150257662, 'num_input_tokens_seen': 3865051136, 'completed': '49.99% (1_843 / 3_687)', 'remaining time': '16:27:42', 'throughput': '8152.85', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 00:47:05,464 >> {'loss': 0.4491, 'grad_norm': 16.989797592163086, 'learning_rate': 5.8709718252536e-07, 'epoch': 0.28315703824247357, 'num_input_tokens_seen': 3867148288, 'completed': '50.01% (1_844 / 3_687)', 'remaining time': '16:27:11', 'throughput': '7999.25', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 00:47:34,705 >> {'loss': 0.338, 'grad_norm': 14.282411575317383, 'learning_rate': 5.866948533583152e-07, 'epoch': 0.28342826145918093, 'num_input_tokens_seen': 3869245440, 'completed': '50.04% (1_845 / 3_687)', 'remaining time': '16:26:34', 'throughput': '8964.86', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 00:48:04,495 >> {'loss': 0.2293, 'grad_norm': 11.778130531311035, 'learning_rate': 5.862924946606487e-07, 'epoch': 0.28369948467588824, 'num_input_tokens_seen': 3871342592, 'completed': '50.07% (1_846 / 3_687)', 'remaining time': '16:25:58', 'throughput': '8799.74', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 00:48:37,247 >> {'loss': 0.4903, 'grad_norm': 17.8021297454834, 'learning_rate': 5.858901067561637e-07, 'epoch': 0.2839707078925956, 'num_input_tokens_seen': 3873439744, 'completed': '50.09% (1_847 / 3_687)', 'remaining time': '16:25:27', 'throughput': '8003.91', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 00:49:08,297 >> {'loss': 0.8826, 'grad_norm': 21.23961067199707, 'learning_rate': 5.854876899686864e-07, 'epoch': 0.28424193110930296, 'num_input_tokens_seen': 3875536896, 'completed': '50.12% (1_848 / 3_687)', 'remaining time': '16:24:53', 'throughput': '8442.69', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 00:49:36,784 >> {'loss': 0.6914, 'grad_norm': 21.922151565551758, 'learning_rate': 5.850852446220666e-07, 'epoch': 0.2845131543260103, 'num_input_tokens_seen': 3877634048, 'completed': '50.15% (1_849 / 3_687)', 'remaining time': '16:24:14', 'throughput': '9202.19', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 00:50:05,884 >> {'loss': 0.3711, 'grad_norm': 15.679913520812988, 'learning_rate': 5.84682771040177e-07, 'epoch': 0.2847843775427177, 'num_input_tokens_seen': 3879731200, 'completed': '50.18% (1_850 / 3_687)', 'remaining time': '16:23:37', 'throughput': '9008.31', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 00:50:37,762 >> {'loss': 0.3313, 'grad_norm': 12.335871696472168, 'learning_rate': 5.842802695469131e-07, 'epoch': 0.285055600759425, 'num_input_tokens_seen': 3881828352, 'completed': '50.20% (1_851 / 3_687)', 'remaining time': '16:23:04', 'throughput': '8223.52', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 00:51:07,502 >> {'loss': 0.3604, 'grad_norm': 13.751896858215332, 'learning_rate': 5.838777404661927e-07, 'epoch': 0.28532682397613235, 'num_input_tokens_seen': 3883925504, 'completed': '50.23% (1_852 / 3_687)', 'remaining time': '16:22:28', 'throughput': '8814.37', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 00:51:35,129 >> {'loss': 0.6989, 'grad_norm': 17.7875919342041, 'learning_rate': 5.83475184121956e-07, 'epoch': 0.2855980471928397, 'num_input_tokens_seen': 3886022656, 'completed': '50.26% (1_853 / 3_687)', 'remaining time': '16:21:48', 'throughput': '9488.68', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 00:52:06,094 >> {'loss': 0.6116, 'grad_norm': 21.652841567993164, 'learning_rate': 5.830726008381648e-07, 'epoch': 0.28586927040954707, 'num_input_tokens_seen': 3888119808, 'completed': '50.28% (1_854 / 3_687)', 'remaining time': '16:21:14', 'throughput': '8465.75', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 00:52:36,090 >> {'loss': 0.4374, 'grad_norm': 17.218547821044922, 'learning_rate': 5.826699909388031e-07, 'epoch': 0.28614049362625443, 'num_input_tokens_seen': 3890216960, 'completed': '50.31% (1_855 / 3_687)', 'remaining time': '16:20:38', 'throughput': '8739.43', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 00:53:07,944 >> {'loss': 0.5781, 'grad_norm': 16.56511116027832, 'learning_rate': 5.822673547478757e-07, 'epoch': 0.28641171684296174, 'num_input_tokens_seen': 3892314112, 'completed': '50.34% (1_856 / 3_687)', 'remaining time': '16:20:05', 'throughput': '8229.42', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 00:53:39,898 >> {'loss': 0.6474, 'grad_norm': 18.990068435668945, 'learning_rate': 5.818646925894092e-07, 'epoch': 0.2866829400596691, 'num_input_tokens_seen': 3894411264, 'completed': '50.37% (1_857 / 3_687)', 'remaining time': '16:19:33', 'throughput': '8203.90', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 00:54:11,311 >> {'loss': 0.2562, 'grad_norm': 9.938685417175293, 'learning_rate': 5.814620047874505e-07, 'epoch': 0.28695416327637646, 'num_input_tokens_seen': 3896508416, 'completed': '50.39% (1_858 / 3_687)', 'remaining time': '16:19:00', 'throughput': '8345.13', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 00:54:41,235 >> {'loss': 0.4856, 'grad_norm': 16.420949935913086, 'learning_rate': 5.810592916660677e-07, 'epoch': 0.2872253864930838, 'num_input_tokens_seen': 3898605568, 'completed': '50.42% (1_859 / 3_687)', 'remaining time': '16:18:24', 'throughput': '8760.16', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 00:55:12,301 >> {'loss': 0.3751, 'grad_norm': 17.336559295654297, 'learning_rate': 5.806565535493489e-07, 'epoch': 0.2874966097097912, 'num_input_tokens_seen': 3900702720, 'completed': '50.45% (1_860 / 3_687)', 'remaining time': '16:17:50', 'throughput': '8439.32', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 00:55:43,127 >> {'loss': 0.3521, 'grad_norm': 12.965970039367676, 'learning_rate': 5.802537907614023e-07, 'epoch': 0.2877678329264985, 'num_input_tokens_seen': 3902799872, 'completed': '50.47% (1_861 / 3_687)', 'remaining time': '16:17:15', 'throughput': '8503.01', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 00:56:13,576 >> {'loss': 0.7629, 'grad_norm': 20.108755111694336, 'learning_rate': 5.798510036263561e-07, 'epoch': 0.28803905614320585, 'num_input_tokens_seen': 3904897024, 'completed': '50.50% (1_862 / 3_687)', 'remaining time': '16:16:40', 'throughput': '8609.27', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 00:56:42,649 >> {'loss': 0.5382, 'grad_norm': 15.406062126159668, 'learning_rate': 5.794481924683581e-07, 'epoch': 0.2883102793599132, 'num_input_tokens_seen': 3906994176, 'completed': '50.53% (1_863 / 3_687)', 'remaining time': '16:16:03', 'throughput': '9016.88', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 00:57:15,971 >> {'loss': 0.4859, 'grad_norm': 21.75050163269043, 'learning_rate': 5.790453576115756e-07, 'epoch': 0.2885815025766206, 'num_input_tokens_seen': 3909091328, 'completed': '50.56% (1_864 / 3_687)', 'remaining time': '16:15:33', 'throughput': '7867.05', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 00:57:46,769 >> {'loss': 0.2867, 'grad_norm': 11.159966468811035, 'learning_rate': 5.786424993801942e-07, 'epoch': 0.28885272579332794, 'num_input_tokens_seen': 3911188480, 'completed': '50.58% (1_865 / 3_687)', 'remaining time': '16:14:59', 'throughput': '8511.71', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 00:58:19,411 >> {'loss': 0.3638, 'grad_norm': 16.282310485839844, 'learning_rate': 5.782396180984194e-07, 'epoch': 0.28912394901003524, 'num_input_tokens_seen': 3913285632, 'completed': '50.61% (1_866 / 3_687)', 'remaining time': '16:14:28', 'throughput': '8030.84', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 00:58:50,984 >> {'loss': 0.4206, 'grad_norm': 15.254396438598633, 'learning_rate': 5.778367140904746e-07, 'epoch': 0.2893951722267426, 'num_input_tokens_seen': 3915382784, 'completed': '50.64% (1_867 / 3_687)', 'remaining time': '16:13:55', 'throughput': '8302.83', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 00:59:23,080 >> {'loss': 0.5837, 'grad_norm': 18.541379928588867, 'learning_rate': 5.774337876806016e-07, 'epoch': 0.28966639544344996, 'num_input_tokens_seen': 3917479936, 'completed': '50.66% (1_868 / 3_687)', 'remaining time': '16:13:22', 'throughput': '8167.46', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 00:59:53,540 >> {'loss': 0.5548, 'grad_norm': 20.998781204223633, 'learning_rate': 5.770308391930601e-07, 'epoch': 0.2899376186601573, 'num_input_tokens_seen': 3919577088, 'completed': '50.69% (1_869 / 3_687)', 'remaining time': '16:12:48', 'throughput': '8606.09', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 01:00:24,165 >> {'loss': 0.428, 'grad_norm': 14.681344985961914, 'learning_rate': 5.766278689521278e-07, 'epoch': 0.2902088418768647, 'num_input_tokens_seen': 3921674240, 'completed': '50.72% (1_870 / 3_687)', 'remaining time': '16:12:13', 'throughput': '8559.95', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 01:00:53,018 >> {'loss': 0.3078, 'grad_norm': 11.808730125427246, 'learning_rate': 5.762248772820999e-07, 'epoch': 0.290480065093572, 'num_input_tokens_seen': 3923771392, 'completed': '50.75% (1_871 / 3_687)', 'remaining time': '16:11:35', 'throughput': '9085.28', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 01:01:24,030 >> {'loss': 0.3758, 'grad_norm': 12.644157409667969, 'learning_rate': 5.758218645072887e-07, 'epoch': 0.29075128831027935, 'num_input_tokens_seen': 3925868544, 'completed': '50.77% (1_872 / 3_687)', 'remaining time': '16:11:01', 'throughput': '8453.17', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 01:01:52,895 >> {'loss': 0.5625, 'grad_norm': 18.32072639465332, 'learning_rate': 5.754188309520235e-07, 'epoch': 0.2910225115269867, 'num_input_tokens_seen': 3927965696, 'completed': '50.80% (1_873 / 3_687)', 'remaining time': '16:10:24', 'throughput': '9081.67', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 01:02:21,630 >> {'loss': 0.6177, 'grad_norm': 20.193140029907227, 'learning_rate': 5.750157769406504e-07, 'epoch': 0.2912937347436941, 'num_input_tokens_seen': 3930062848, 'completed': '50.83% (1_874 / 3_687)', 'remaining time': '16:09:46', 'throughput': '9122.64', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 01:02:50,497 >> {'loss': 0.7787, 'grad_norm': 20.246875762939453, 'learning_rate': 5.74612702797532e-07, 'epoch': 0.2915649579604014, 'num_input_tokens_seen': 3932160000, 'completed': '50.85% (1_875 / 3_687)', 'remaining time': '16:09:08', 'throughput': '9081.18', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 01:03:17,710 >> {'loss': 0.7121, 'grad_norm': 44.87726974487305, 'learning_rate': 5.74209608847047e-07, 'epoch': 0.29183618117710874, 'num_input_tokens_seen': 3934257152, 'completed': '50.88% (1_876 / 3_687)', 'remaining time': '16:08:28', 'throughput': '9632.91', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 01:03:47,425 >> {'loss': 0.6667, 'grad_norm': 22.28316879272461, 'learning_rate': 5.7380649541359e-07, 'epoch': 0.2921074043938161, 'num_input_tokens_seen': 3936354304, 'completed': '50.91% (1_877 / 3_687)', 'remaining time': '16:07:52', 'throughput': '8821.97', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 01:04:17,483 >> {'loss': 0.3718, 'grad_norm': 12.953632354736328, 'learning_rate': 5.734033628215714e-07, 'epoch': 0.29237862761052347, 'num_input_tokens_seen': 3938451456, 'completed': '50.94% (1_878 / 3_687)', 'remaining time': '16:07:17', 'throughput': '8721.50', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 01:04:48,857 >> {'loss': 0.6267, 'grad_norm': 18.659269332885742, 'learning_rate': 5.730002113954169e-07, 'epoch': 0.29264985082723083, 'num_input_tokens_seen': 3940548608, 'completed': '50.96% (1_879 / 3_687)', 'remaining time': '16:06:43', 'throughput': '8355.36', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 01:05:19,595 >> {'loss': 0.7124, 'grad_norm': 24.658376693725586, 'learning_rate': 5.725970414595675e-07, 'epoch': 0.29292107404393813, 'num_input_tokens_seen': 3942645760, 'completed': '50.99% (1_880 / 3_687)', 'remaining time': '16:06:09', 'throughput': '8528.26', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 01:05:48,903 >> {'loss': 0.5809, 'grad_norm': 15.565204620361328, 'learning_rate': 5.721938533384791e-07, 'epoch': 0.2931922972606455, 'num_input_tokens_seen': 3944742912, 'completed': '51.02% (1_881 / 3_687)', 'remaining time': '16:05:32', 'throughput': '8944.48', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 01:06:19,440 >> {'loss': 0.4206, 'grad_norm': 16.507081985473633, 'learning_rate': 5.717906473566219e-07, 'epoch': 0.29346352047735286, 'num_input_tokens_seen': 3946840064, 'completed': '51.04% (1_882 / 3_687)', 'remaining time': '16:04:58', 'throughput': '8584.44', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 01:06:49,039 >> {'loss': 0.6355, 'grad_norm': 19.66336441040039, 'learning_rate': 5.71387423838481e-07, 'epoch': 0.2937347436940602, 'num_input_tokens_seen': 3948937216, 'completed': '51.07% (1_883 / 3_687)', 'remaining time': '16:04:21', 'throughput': '8856.64', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 01:07:19,866 >> {'loss': 0.3339, 'grad_norm': 13.438267707824707, 'learning_rate': 5.709841831085553e-07, 'epoch': 0.2940059669107676, 'num_input_tokens_seen': 3951034368, 'completed': '51.10% (1_884 / 3_687)', 'remaining time': '16:03:47', 'throughput': '8503.59', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 01:07:51,507 >> {'loss': 0.3658, 'grad_norm': 12.036945343017578, 'learning_rate': 5.705809254913576e-07, 'epoch': 0.2942771901274749, 'num_input_tokens_seen': 3953131520, 'completed': '51.13% (1_885 / 3_687)', 'remaining time': '16:03:14', 'throughput': '8284.88', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 01:08:19,303 >> {'loss': 0.6545, 'grad_norm': 19.046716690063477, 'learning_rate': 5.70177651311414e-07, 'epoch': 0.29454841334418225, 'num_input_tokens_seen': 3955228672, 'completed': '51.15% (1_886 / 3_687)', 'remaining time': '16:02:35', 'throughput': '9431.01', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 01:08:51,718 >> {'loss': 0.3465, 'grad_norm': 16.657411575317383, 'learning_rate': 5.697743608932646e-07, 'epoch': 0.2948196365608896, 'num_input_tokens_seen': 3957325824, 'completed': '51.18% (1_887 / 3_687)', 'remaining time': '16:02:04', 'throughput': '8087.16', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 01:09:21,507 >> {'loss': 1.0207, 'grad_norm': 28.270238876342773, 'learning_rate': 5.693710545614621e-07, 'epoch': 0.29509085977759697, 'num_input_tokens_seen': 3959422976, 'completed': '51.21% (1_888 / 3_687)', 'remaining time': '16:01:28', 'throughput': '8800.25', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 01:09:50,136 >> {'loss': 0.6864, 'grad_norm': 18.105026245117188, 'learning_rate': 5.689677326405719e-07, 'epoch': 0.29536208299430433, 'num_input_tokens_seen': 3961520128, 'completed': '51.23% (1_889 / 3_687)', 'remaining time': '16:00:50', 'throughput': '9156.51', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 01:10:20,526 >> {'loss': 0.8733, 'grad_norm': 22.831960678100586, 'learning_rate': 5.685643954551722e-07, 'epoch': 0.29563330621101164, 'num_input_tokens_seen': 3963617280, 'completed': '51.26% (1_890 / 3_687)', 'remaining time': '16:00:15', 'throughput': '8626.07', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 01:10:51,972 >> {'loss': 0.5093, 'grad_norm': 18.551103591918945, 'learning_rate': 5.681610433298535e-07, 'epoch': 0.295904529427719, 'num_input_tokens_seen': 3965714432, 'completed': '51.29% (1_891 / 3_687)', 'remaining time': '15:59:42', 'throughput': '8336.15', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 01:11:24,618 >> {'loss': 0.4713, 'grad_norm': 15.365251541137695, 'learning_rate': 5.677576765892182e-07, 'epoch': 0.29617575264442636, 'num_input_tokens_seen': 3967811584, 'completed': '51.32% (1_892 / 3_687)', 'remaining time': '15:59:11', 'throughput': '8030.02', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 01:11:54,766 >> {'loss': 0.4017, 'grad_norm': 16.42351722717285, 'learning_rate': 5.673542955578806e-07, 'epoch': 0.2964469758611337, 'num_input_tokens_seen': 3969908736, 'completed': '51.34% (1_893 / 3_687)', 'remaining time': '15:58:36', 'throughput': '8695.15', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 01:12:25,203 >> {'loss': 0.3471, 'grad_norm': 15.9588623046875, 'learning_rate': 5.669509005604663e-07, 'epoch': 0.2967181990778411, 'num_input_tokens_seen': 3972005888, 'completed': '51.37% (1_894 / 3_687)', 'remaining time': '15:58:01', 'throughput': '8612.55', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 01:12:55,073 >> {'loss': 0.7455, 'grad_norm': 21.854177474975586, 'learning_rate': 5.665474919216122e-07, 'epoch': 0.2969894222945484, 'num_input_tokens_seen': 3974103040, 'completed': '51.40% (1_895 / 3_687)', 'remaining time': '15:57:26', 'throughput': '8776.36', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 01:13:25,290 >> {'loss': 0.4565, 'grad_norm': 15.875153541564941, 'learning_rate': 5.661440699659663e-07, 'epoch': 0.29726064551125575, 'num_input_tokens_seen': 3976200192, 'completed': '51.42% (1_896 / 3_687)', 'remaining time': '15:56:51', 'throughput': '8675.28', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 01:13:54,176 >> {'loss': 0.3644, 'grad_norm': 13.967791557312012, 'learning_rate': 5.657406350181872e-07, 'epoch': 0.2975318687279631, 'num_input_tokens_seen': 3978297344, 'completed': '51.45% (1_897 / 3_687)', 'remaining time': '15:56:13', 'throughput': '9075.24', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 01:14:24,610 >> {'loss': 0.7488, 'grad_norm': 22.632991790771484, 'learning_rate': 5.65337187402944e-07, 'epoch': 0.2978030919446705, 'num_input_tokens_seen': 3980394496, 'completed': '51.48% (1_898 / 3_687)', 'remaining time': '15:55:39', 'throughput': '8613.54', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 01:14:57,226 >> {'loss': 0.5608, 'grad_norm': 35.31967544555664, 'learning_rate': 5.64933727444916e-07, 'epoch': 0.29807431516137783, 'num_input_tokens_seen': 3982491648, 'completed': '51.51% (1_899 / 3_687)', 'remaining time': '15:55:08', 'throughput': '8037.24', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 01:15:27,162 >> {'loss': 0.3702, 'grad_norm': 13.531403541564941, 'learning_rate': 5.645302554687925e-07, 'epoch': 0.29834553837808514, 'num_input_tokens_seen': 3984588800, 'completed': '51.53% (1_900 / 3_687)', 'remaining time': '15:54:32', 'throughput': '8756.63', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 01:15:57,748 >> {'loss': 0.4089, 'grad_norm': 15.457035064697266, 'learning_rate': 5.641267717992723e-07, 'epoch': 0.2986167615947925, 'num_input_tokens_seen': 3986685952, 'completed': '51.56% (1_901 / 3_687)', 'remaining time': '15:53:58', 'throughput': '8570.82', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 01:16:29,760 >> {'loss': 0.4136, 'grad_norm': 15.16627311706543, 'learning_rate': 5.637232767610637e-07, 'epoch': 0.29888798481149986, 'num_input_tokens_seen': 3988783104, 'completed': '51.59% (1_902 / 3_687)', 'remaining time': '15:53:26', 'throughput': '8188.88', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 01:17:02,471 >> {'loss': 0.4861, 'grad_norm': 15.52268123626709, 'learning_rate': 5.63319770678884e-07, 'epoch': 0.2991592080282072, 'num_input_tokens_seen': 3990880256, 'completed': '51.61% (1_903 / 3_687)', 'remaining time': '15:52:55', 'throughput': '8014.06', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 01:17:34,357 >> {'loss': 0.3985, 'grad_norm': 14.946885108947754, 'learning_rate': 5.6291625387746e-07, 'epoch': 0.2994304312449146, 'num_input_tokens_seen': 3992977408, 'completed': '51.64% (1_904 / 3_687)', 'remaining time': '15:52:22', 'throughput': '8221.17', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 01:18:03,808 >> {'loss': 0.3807, 'grad_norm': 18.582761764526367, 'learning_rate': 5.625127266815263e-07, 'epoch': 0.2997016544616219, 'num_input_tokens_seen': 3995074560, 'completed': '51.67% (1_905 / 3_687)', 'remaining time': '15:51:46', 'throughput': '8901.22', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 01:18:35,236 >> {'loss': 0.3393, 'grad_norm': 17.98885726928711, 'learning_rate': 5.621091894158261e-07, 'epoch': 0.29997287767832925, 'num_input_tokens_seen': 3997171712, 'completed': '51.70% (1_906 / 3_687)', 'remaining time': '15:51:13', 'throughput': '8340.90', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 01:19:02,755 >> {'loss': 0.5337, 'grad_norm': 17.400163650512695, 'learning_rate': 5.617056424051113e-07, 'epoch': 0.3002441008950366, 'num_input_tokens_seen': 3999268864, 'completed': '51.72% (1_907 / 3_687)', 'remaining time': '15:50:34', 'throughput': '9526.21', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 01:19:34,580 >> {'loss': 0.4942, 'grad_norm': 20.89246368408203, 'learning_rate': 5.613020859741408e-07, 'epoch': 0.300515324111744, 'num_input_tokens_seen': 4001366016, 'completed': '51.75% (1_908 / 3_687)', 'remaining time': '15:50:01', 'throughput': '8236.96', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 01:20:08,771 >> {'loss': 0.6841, 'grad_norm': 22.6678524017334, 'learning_rate': 5.608985204476817e-07, 'epoch': 0.30078654732845134, 'num_input_tokens_seen': 4003463168, 'completed': '51.78% (1_909 / 3_687)', 'remaining time': '15:49:33', 'throughput': '7667.13', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 01:20:41,786 >> {'loss': 0.5315, 'grad_norm': 29.596494674682617, 'learning_rate': 5.60494946150508e-07, 'epoch': 0.30105777054515864, 'num_input_tokens_seen': 4005560320, 'completed': '51.80% (1_910 / 3_687)', 'remaining time': '15:49:02', 'throughput': '7940.15', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 01:21:13,288 >> {'loss': 0.4301, 'grad_norm': 16.15056037902832, 'learning_rate': 5.600913634074009e-07, 'epoch': 0.301328993761866, 'num_input_tokens_seen': 4007657472, 'completed': '51.83% (1_911 / 3_687)', 'remaining time': '15:48:29', 'throughput': '8321.40', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 01:21:43,918 >> {'loss': 0.4657, 'grad_norm': 16.99407196044922, 'learning_rate': 5.596877725431487e-07, 'epoch': 0.30160021697857337, 'num_input_tokens_seen': 4009754624, 'completed': '51.86% (1_912 / 3_687)', 'remaining time': '15:47:55', 'throughput': '8558.38', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 01:22:14,060 >> {'loss': 0.313, 'grad_norm': 13.268138885498047, 'learning_rate': 5.592841738825457e-07, 'epoch': 0.3018714401952807, 'num_input_tokens_seen': 4011851776, 'completed': '51.89% (1_913 / 3_687)', 'remaining time': '15:47:20', 'throughput': '8696.91', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 01:22:43,982 >> {'loss': 0.3853, 'grad_norm': 16.610029220581055, 'learning_rate': 5.588805677503928e-07, 'epoch': 0.3021426634119881, 'num_input_tokens_seen': 4013948928, 'completed': '51.91% (1_914 / 3_687)', 'remaining time': '15:46:44', 'throughput': '8760.95', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 01:23:19,793 >> {'loss': 0.4144, 'grad_norm': 14.709389686584473, 'learning_rate': 5.58476954471497e-07, 'epoch': 0.3024138866286954, 'num_input_tokens_seen': 4016046080, 'completed': '51.94% (1_915 / 3_687)', 'remaining time': '15:46:18', 'throughput': '7320.14', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 01:23:51,379 >> {'loss': 0.6721, 'grad_norm': 18.49999237060547, 'learning_rate': 5.580733343706708e-07, 'epoch': 0.30268510984540276, 'num_input_tokens_seen': 4018143232, 'completed': '51.97% (1_916 / 3_687)', 'remaining time': '15:45:46', 'throughput': '8299.56', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 01:24:22,240 >> {'loss': 0.4901, 'grad_norm': 17.968942642211914, 'learning_rate': 5.576697077727323e-07, 'epoch': 0.3029563330621101, 'num_input_tokens_seen': 4020240384, 'completed': '51.99% (1_917 / 3_687)', 'remaining time': '15:45:12', 'throughput': '8494.17', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 01:24:52,410 >> {'loss': 0.4983, 'grad_norm': 18.4853515625, 'learning_rate': 5.572660750025049e-07, 'epoch': 0.3032275562788175, 'num_input_tokens_seen': 4022337536, 'completed': '52.02% (1_918 / 3_687)', 'remaining time': '15:44:37', 'throughput': '8688.89', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 01:25:25,310 >> {'loss': 0.3819, 'grad_norm': 15.520868301391602, 'learning_rate': 5.568624363848166e-07, 'epoch': 0.30349877949552484, 'num_input_tokens_seen': 4024434688, 'completed': '52.05% (1_919 / 3_687)', 'remaining time': '15:44:06', 'throughput': '7967.96', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 01:25:56,294 >> {'loss': 0.2858, 'grad_norm': 10.888456344604492, 'learning_rate': 5.564587922445008e-07, 'epoch': 0.30377000271223215, 'num_input_tokens_seen': 4026531840, 'completed': '52.07% (1_920 / 3_687)', 'remaining time': '15:43:32', 'throughput': '8460.63', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 01:26:30,073 >> {'loss': 0.3409, 'grad_norm': 13.872169494628906, 'learning_rate': 5.560551429063949e-07, 'epoch': 0.3040412259289395, 'num_input_tokens_seen': 4028628992, 'completed': '52.10% (1_921 / 3_687)', 'remaining time': '15:43:03', 'throughput': '7760.66', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 01:26:58,298 >> {'loss': 0.7254, 'grad_norm': 22.901479721069336, 'learning_rate': 5.556514886953403e-07, 'epoch': 0.30431244914564687, 'num_input_tokens_seen': 4030726144, 'completed': '52.13% (1_922 / 3_687)', 'remaining time': '15:42:25', 'throughput': '9287.51', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 01:27:30,150 >> {'loss': 0.4352, 'grad_norm': 15.373479843139648, 'learning_rate': 5.552478299361826e-07, 'epoch': 0.30458367236235423, 'num_input_tokens_seen': 4032823296, 'completed': '52.16% (1_923 / 3_687)', 'remaining time': '15:41:53', 'throughput': '8230.05', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 01:28:00,922 >> {'loss': 0.6943, 'grad_norm': 17.427900314331055, 'learning_rate': 5.548441669537712e-07, 'epoch': 0.3048548955790616, 'num_input_tokens_seen': 4034920448, 'completed': '52.18% (1_924 / 3_687)', 'remaining time': '15:41:19', 'throughput': '8518.99', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 01:28:30,047 >> {'loss': 0.5922, 'grad_norm': 20.171907424926758, 'learning_rate': 5.544405000729584e-07, 'epoch': 0.3051261187957689, 'num_input_tokens_seen': 4037017600, 'completed': '52.21% (1_925 / 3_687)', 'remaining time': '15:40:42', 'throughput': '9000.74', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 01:28:59,315 >> {'loss': 0.4632, 'grad_norm': 15.804996490478516, 'learning_rate': 5.540368296186002e-07, 'epoch': 0.30539734201247626, 'num_input_tokens_seen': 4039114752, 'completed': '52.24% (1_926 / 3_687)', 'remaining time': '15:40:06', 'throughput': '8956.51', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 01:29:29,123 >> {'loss': 0.3587, 'grad_norm': 13.54218864440918, 'learning_rate': 5.53633155915555e-07, 'epoch': 0.3056685652291836, 'num_input_tokens_seen': 4041211904, 'completed': '52.26% (1_927 / 3_687)', 'remaining time': '15:39:30', 'throughput': '8794.61', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 01:30:00,195 >> {'loss': 0.8946, 'grad_norm': 24.575578689575195, 'learning_rate': 5.532294792886843e-07, 'epoch': 0.305939788445891, 'num_input_tokens_seen': 4043309056, 'completed': '52.29% (1_928 / 3_687)', 'remaining time': '15:38:57', 'throughput': '8436.64', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 01:30:31,640 >> {'loss': 0.4468, 'grad_norm': 18.244264602661133, 'learning_rate': 5.528258000628518e-07, 'epoch': 0.30621101166259834, 'num_input_tokens_seen': 4045406208, 'completed': '52.32% (1_929 / 3_687)', 'remaining time': '15:38:24', 'throughput': '8336.50', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 01:30:59,061 >> {'loss': 0.5916, 'grad_norm': 18.432344436645508, 'learning_rate': 5.524221185629224e-07, 'epoch': 0.30648223487930565, 'num_input_tokens_seen': 4047503360, 'completed': '52.35% (1_930 / 3_687)', 'remaining time': '15:37:45', 'throughput': '9559.85', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 01:31:29,188 >> {'loss': 0.428, 'grad_norm': 14.628120422363281, 'learning_rate': 5.520184351137646e-07, 'epoch': 0.306753458096013, 'num_input_tokens_seen': 4049600512, 'completed': '52.37% (1_931 / 3_687)', 'remaining time': '15:37:10', 'throughput': '8701.52', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 01:32:01,545 >> {'loss': 0.3338, 'grad_norm': 15.325323104858398, 'learning_rate': 5.51614750040247e-07, 'epoch': 0.30702468131272037, 'num_input_tokens_seen': 4051697664, 'completed': '52.40% (1_932 / 3_687)', 'remaining time': '15:36:38', 'throughput': '8101.45', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 01:32:31,887 >> {'loss': 0.5334, 'grad_norm': 18.96306037902832, 'learning_rate': 5.512110636672396e-07, 'epoch': 0.30729590452942773, 'num_input_tokens_seen': 4053794816, 'completed': '52.43% (1_933 / 3_687)', 'remaining time': '15:36:03', 'throughput': '8639.56', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 01:33:02,432 >> {'loss': 0.405, 'grad_norm': 16.209426879882812, 'learning_rate': 5.508073763196146e-07, 'epoch': 0.3075671277461351, 'num_input_tokens_seen': 4055891968, 'completed': '52.45% (1_934 / 3_687)', 'remaining time': '15:35:29', 'throughput': '8582.26', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 01:33:32,336 >> {'loss': 0.4682, 'grad_norm': 17.50925064086914, 'learning_rate': 5.504036883222438e-07, 'epoch': 0.3078383509628424, 'num_input_tokens_seen': 4057989120, 'completed': '52.48% (1_935 / 3_687)', 'remaining time': '15:34:54', 'throughput': '8766.26', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 01:34:07,142 >> {'loss': 0.6966, 'grad_norm': 21.680133819580078, 'learning_rate': 5.5e-07, 'epoch': 0.30810957417954976, 'num_input_tokens_seen': 4060086272, 'completed': '52.51% (1_936 / 3_687)', 'remaining time': '15:34:26', 'throughput': '7531.56', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 01:34:36,619 >> {'loss': 0.3213, 'grad_norm': 16.945390701293945, 'learning_rate': 5.495963116777562e-07, 'epoch': 0.3083807973962571, 'num_input_tokens_seen': 4062183424, 'completed': '52.54% (1_937 / 3_687)', 'remaining time': '15:33:50', 'throughput': '8893.10', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 01:35:07,271 >> {'loss': 0.6936, 'grad_norm': 19.299571990966797, 'learning_rate': 5.491926236803854e-07, 'epoch': 0.3086520206129645, 'num_input_tokens_seen': 4064280576, 'completed': '52.56% (1_938 / 3_687)', 'remaining time': '15:33:16', 'throughput': '8552.37', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 01:35:41,302 >> {'loss': 0.5167, 'grad_norm': 16.683467864990234, 'learning_rate': 5.487889363327603e-07, 'epoch': 0.30892324382967185, 'num_input_tokens_seen': 4066377728, 'completed': '52.59% (1_939 / 3_687)', 'remaining time': '15:32:47', 'throughput': '7703.28', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 01:36:14,584 >> {'loss': 0.308, 'grad_norm': 11.593538284301758, 'learning_rate': 5.483852499597532e-07, 'epoch': 0.30919446704637915, 'num_input_tokens_seen': 4068474880, 'completed': '52.62% (1_940 / 3_687)', 'remaining time': '15:32:17', 'throughput': '7876.25', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 01:36:45,496 >> {'loss': 0.6182, 'grad_norm': 21.42290687561035, 'learning_rate': 5.479815648862355e-07, 'epoch': 0.3094656902630865, 'num_input_tokens_seen': 4070572032, 'completed': '52.64% (1_941 / 3_687)', 'remaining time': '15:31:43', 'throughput': '8480.48', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 01:37:17,417 >> {'loss': 0.2313, 'grad_norm': 10.256348609924316, 'learning_rate': 5.475778814370776e-07, 'epoch': 0.3097369134797939, 'num_input_tokens_seen': 4072669184, 'completed': '52.67% (1_942 / 3_687)', 'remaining time': '15:31:11', 'throughput': '8212.25', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 01:37:48,371 >> {'loss': 0.3522, 'grad_norm': 12.884416580200195, 'learning_rate': 5.471741999371483e-07, 'epoch': 0.31000813669650124, 'num_input_tokens_seen': 4074766336, 'completed': '52.70% (1_943 / 3_687)', 'remaining time': '15:30:38', 'throughput': '8468.74', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 01:38:17,584 >> {'loss': 0.3817, 'grad_norm': 15.000761032104492, 'learning_rate': 5.467705207113156e-07, 'epoch': 0.3102793599132086, 'num_input_tokens_seen': 4076863488, 'completed': '52.73% (1_944 / 3_687)', 'remaining time': '15:30:01', 'throughput': '8973.49', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 01:38:47,899 >> {'loss': 0.5017, 'grad_norm': 18.647953033447266, 'learning_rate': 5.46366844084445e-07, 'epoch': 0.3105505831299159, 'num_input_tokens_seen': 4078960640, 'completed': '52.75% (1_945 / 3_687)', 'remaining time': '15:29:27', 'throughput': '8647.24', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 01:39:15,118 >> {'loss': 0.4558, 'grad_norm': 16.23006820678711, 'learning_rate': 5.459631703813998e-07, 'epoch': 0.31082180634662326, 'num_input_tokens_seen': 4081057792, 'completed': '52.78% (1_946 / 3_687)', 'remaining time': '15:28:47', 'throughput': '9630.91', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 01:39:45,536 >> {'loss': 0.3032, 'grad_norm': 14.229667663574219, 'learning_rate': 5.455594999270416e-07, 'epoch': 0.3110930295633306, 'num_input_tokens_seen': 4083154944, 'completed': '52.81% (1_947 / 3_687)', 'remaining time': '15:28:13', 'throughput': '8618.11', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 01:40:17,046 >> {'loss': 0.5615, 'grad_norm': 22.243261337280273, 'learning_rate': 5.451558330462289e-07, 'epoch': 0.311364252780038, 'num_input_tokens_seen': 4085252096, 'completed': '52.83% (1_948 / 3_687)', 'remaining time': '15:27:40', 'throughput': '8319.42', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 01:40:46,459 >> {'loss': 0.4128, 'grad_norm': 16.879133224487305, 'learning_rate': 5.447521700638174e-07, 'epoch': 0.31163547599674535, 'num_input_tokens_seen': 4087349248, 'completed': '52.86% (1_949 / 3_687)', 'remaining time': '15:27:04', 'throughput': '8912.63', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 01:41:17,013 >> {'loss': 0.4147, 'grad_norm': 17.294158935546875, 'learning_rate': 5.443485113046597e-07, 'epoch': 0.31190669921345265, 'num_input_tokens_seen': 4089446400, 'completed': '52.89% (1_950 / 3_687)', 'remaining time': '15:26:30', 'throughput': '8579.75', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 01:41:46,907 >> {'loss': 0.3165, 'grad_norm': 12.599308967590332, 'learning_rate': 5.439448570936052e-07, 'epoch': 0.31217792243016, 'num_input_tokens_seen': 4091543552, 'completed': '52.92% (1_951 / 3_687)', 'remaining time': '15:25:55', 'throughput': '8769.13', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 01:42:16,783 >> {'loss': 0.4091, 'grad_norm': 16.82218360900879, 'learning_rate': 5.435412077554991e-07, 'epoch': 0.3124491456468674, 'num_input_tokens_seen': 4093640704, 'completed': '52.94% (1_952 / 3_687)', 'remaining time': '15:25:20', 'throughput': '8774.31', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 01:42:45,757 >> {'loss': 0.4083, 'grad_norm': 15.809752464294434, 'learning_rate': 5.431375636151833e-07, 'epoch': 0.31272036886357474, 'num_input_tokens_seen': 4095737856, 'completed': '52.97% (1_953 / 3_687)', 'remaining time': '15:24:43', 'throughput': '9047.65', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 01:43:15,094 >> {'loss': 0.8117, 'grad_norm': 24.349023818969727, 'learning_rate': 5.427339249974952e-07, 'epoch': 0.3129915920802821, 'num_input_tokens_seen': 4097835008, 'completed': '53.00% (1_954 / 3_687)', 'remaining time': '15:24:07', 'throughput': '8935.57', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 01:43:45,733 >> {'loss': 0.4125, 'grad_norm': 15.348997116088867, 'learning_rate': 5.423302922272677e-07, 'epoch': 0.3132628152969894, 'num_input_tokens_seen': 4099932160, 'completed': '53.02% (1_955 / 3_687)', 'remaining time': '15:23:33', 'throughput': '8555.93', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 01:44:17,696 >> {'loss': 0.6195, 'grad_norm': 21.631223678588867, 'learning_rate': 5.419266656293293e-07, 'epoch': 0.31353403851369677, 'num_input_tokens_seen': 4102029312, 'completed': '53.05% (1_956 / 3_687)', 'remaining time': '15:23:01', 'throughput': '8201.25', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 01:44:46,115 >> {'loss': 0.4847, 'grad_norm': 17.50019645690918, 'learning_rate': 5.415230455285031e-07, 'epoch': 0.31380526173040413, 'num_input_tokens_seen': 4104126464, 'completed': '53.08% (1_957 / 3_687)', 'remaining time': '15:22:24', 'throughput': '9224.39', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 01:45:16,497 >> {'loss': 0.3183, 'grad_norm': 12.668447494506836, 'learning_rate': 5.411194322496073e-07, 'epoch': 0.3140764849471115, 'num_input_tokens_seen': 4106223616, 'completed': '53.11% (1_958 / 3_687)', 'remaining time': '15:21:49', 'throughput': '8628.37', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 01:45:47,586 >> {'loss': 0.2835, 'grad_norm': 14.355048179626465, 'learning_rate': 5.407158261174543e-07, 'epoch': 0.3143477081638188, 'num_input_tokens_seen': 4108320768, 'completed': '53.13% (1_959 / 3_687)', 'remaining time': '15:21:16', 'throughput': '8432.00', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 01:46:21,705 >> {'loss': 0.2686, 'grad_norm': 13.801246643066406, 'learning_rate': 5.403122274568514e-07, 'epoch': 0.31461893138052616, 'num_input_tokens_seen': 4110417920, 'completed': '53.16% (1_960 / 3_687)', 'remaining time': '15:20:47', 'throughput': '7683.12', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 01:46:51,022 >> {'loss': 0.8419, 'grad_norm': 23.078720092773438, 'learning_rate': 5.399086365925991e-07, 'epoch': 0.3148901545972335, 'num_input_tokens_seen': 4112515072, 'completed': '53.19% (1_961 / 3_687)', 'remaining time': '15:20:11', 'throughput': '8941.68', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 01:47:23,468 >> {'loss': 0.3, 'grad_norm': 13.70639419555664, 'learning_rate': 5.395050538494922e-07, 'epoch': 0.3151613778139409, 'num_input_tokens_seen': 4114612224, 'completed': '53.21% (1_962 / 3_687)', 'remaining time': '15:19:40', 'throughput': '8079.48', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 01:47:52,401 >> {'loss': 0.2514, 'grad_norm': 11.593742370605469, 'learning_rate': 5.391014795523184e-07, 'epoch': 0.31543260103064824, 'num_input_tokens_seen': 4116709376, 'completed': '53.24% (1_963 / 3_687)', 'remaining time': '15:19:03', 'throughput': '9060.25', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 01:48:26,541 >> {'loss': 0.5374, 'grad_norm': 17.700437545776367, 'learning_rate': 5.386979140258592e-07, 'epoch': 0.31570382424735555, 'num_input_tokens_seen': 4118806528, 'completed': '53.27% (1_964 / 3_687)', 'remaining time': '15:18:35', 'throughput': '7678.57', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 01:48:58,673 >> {'loss': 0.3823, 'grad_norm': 13.762934684753418, 'learning_rate': 5.382943575948888e-07, 'epoch': 0.3159750474640629, 'num_input_tokens_seen': 4120903680, 'completed': '53.30% (1_965 / 3_687)', 'remaining time': '15:18:03', 'throughput': '8158.27', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 01:49:30,071 >> {'loss': 0.5349, 'grad_norm': 21.619094848632812, 'learning_rate': 5.378908105841738e-07, 'epoch': 0.31624627068077027, 'num_input_tokens_seen': 4123000832, 'completed': '53.32% (1_966 / 3_687)', 'remaining time': '15:17:30', 'throughput': '8349.31', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 01:50:02,621 >> {'loss': 0.4939, 'grad_norm': 16.5258731842041, 'learning_rate': 5.374872733184737e-07, 'epoch': 0.31651749389747763, 'num_input_tokens_seen': 4125097984, 'completed': '53.35% (1_967 / 3_687)', 'remaining time': '15:16:59', 'throughput': '8053.60', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 01:50:30,809 >> {'loss': 0.6686, 'grad_norm': 19.478639602661133, 'learning_rate': 5.370837461225402e-07, 'epoch': 0.316788717114185, 'num_input_tokens_seen': 4127195136, 'completed': '53.38% (1_968 / 3_687)', 'remaining time': '15:16:21', 'throughput': '9299.74', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 01:51:01,494 >> {'loss': 0.3994, 'grad_norm': 15.19251537322998, 'learning_rate': 5.36680229321116e-07, 'epoch': 0.3170599403308923, 'num_input_tokens_seen': 4129292288, 'completed': '53.40% (1_969 / 3_687)', 'remaining time': '15:15:47', 'throughput': '8543.14', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 01:51:30,546 >> {'loss': 0.5067, 'grad_norm': 18.16364288330078, 'learning_rate': 5.362767232389365e-07, 'epoch': 0.31733116354759966, 'num_input_tokens_seen': 4131389440, 'completed': '53.43% (1_970 / 3_687)', 'remaining time': '15:15:11', 'throughput': '9023.02', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 01:52:00,985 >> {'loss': 0.3757, 'grad_norm': 13.165485382080078, 'learning_rate': 5.358732282007278e-07, 'epoch': 0.317602386764307, 'num_input_tokens_seen': 4133486592, 'completed': '53.46% (1_971 / 3_687)', 'remaining time': '15:14:37', 'throughput': '8612.23', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 01:52:33,475 >> {'loss': 0.5601, 'grad_norm': 17.412708282470703, 'learning_rate': 5.354697445312074e-07, 'epoch': 0.3178736099810144, 'num_input_tokens_seen': 4135583744, 'completed': '53.49% (1_972 / 3_687)', 'remaining time': '15:14:06', 'throughput': '8068.39', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 01:53:03,442 >> {'loss': 0.6123, 'grad_norm': 16.720489501953125, 'learning_rate': 5.35066272555084e-07, 'epoch': 0.31814483319772174, 'num_input_tokens_seen': 4137680896, 'completed': '53.51% (1_973 / 3_687)', 'remaining time': '15:13:31', 'throughput': '8747.80', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 01:53:35,586 >> {'loss': 0.401, 'grad_norm': 16.069841384887695, 'learning_rate': 5.346628125970562e-07, 'epoch': 0.31841605641442905, 'num_input_tokens_seen': 4139778048, 'completed': '53.54% (1_974 / 3_687)', 'remaining time': '15:12:59', 'throughput': '8155.28', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 01:54:06,934 >> {'loss': 0.6885, 'grad_norm': 25.234981536865234, 'learning_rate': 5.342593649818129e-07, 'epoch': 0.3186872796311364, 'num_input_tokens_seen': 4141875200, 'completed': '53.57% (1_975 / 3_687)', 'remaining time': '15:12:26', 'throughput': '8362.40', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 01:54:38,555 >> {'loss': 0.4314, 'grad_norm': 14.604377746582031, 'learning_rate': 5.338559300340338e-07, 'epoch': 0.3189585028478438, 'num_input_tokens_seen': 4143972352, 'completed': '53.59% (1_976 / 3_687)', 'remaining time': '15:11:54', 'throughput': '8290.30', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 01:55:10,125 >> {'loss': 0.4862, 'grad_norm': 17.683347702026367, 'learning_rate': 5.334525080783879e-07, 'epoch': 0.31922972606455113, 'num_input_tokens_seen': 4146069504, 'completed': '53.62% (1_977 / 3_687)', 'remaining time': '15:11:21', 'throughput': '8303.52', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 01:55:40,428 >> {'loss': 0.2884, 'grad_norm': 15.543233871459961, 'learning_rate': 5.330490994395338e-07, 'epoch': 0.3195009492812585, 'num_input_tokens_seen': 4148166656, 'completed': '53.65% (1_978 / 3_687)', 'remaining time': '15:10:47', 'throughput': '8650.62', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 01:56:11,200 >> {'loss': 0.5094, 'grad_norm': 16.377304077148438, 'learning_rate': 5.326457044421195e-07, 'epoch': 0.3197721724979658, 'num_input_tokens_seen': 4150263808, 'completed': '53.68% (1_979 / 3_687)', 'remaining time': '15:10:13', 'throughput': '8519.08', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 01:56:42,318 >> {'loss': 0.4386, 'grad_norm': 14.058331489562988, 'learning_rate': 5.322423234107818e-07, 'epoch': 0.32004339571467316, 'num_input_tokens_seen': 4152360960, 'completed': '53.70% (1_980 / 3_687)', 'remaining time': '15:09:40', 'throughput': '8424.23', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 01:57:15,401 >> {'loss': 0.5509, 'grad_norm': 20.853776931762695, 'learning_rate': 5.318389566701465e-07, 'epoch': 0.3203146189313805, 'num_input_tokens_seen': 4154458112, 'completed': '53.73% (1_981 / 3_687)', 'remaining time': '15:09:09', 'throughput': '7923.80', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 01:57:44,957 >> {'loss': 0.403, 'grad_norm': 17.144161224365234, 'learning_rate': 5.314356045448278e-07, 'epoch': 0.3205858421480879, 'num_input_tokens_seen': 4156555264, 'completed': '53.76% (1_982 / 3_687)', 'remaining time': '15:08:34', 'throughput': '8869.31', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 01:58:14,983 >> {'loss': 0.4065, 'grad_norm': 17.98135757446289, 'learning_rate': 5.310322673594282e-07, 'epoch': 0.32085706536479525, 'num_input_tokens_seen': 4158652416, 'completed': '53.78% (1_983 / 3_687)', 'remaining time': '15:07:59', 'throughput': '8730.64', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 01:58:46,692 >> {'loss': 0.2229, 'grad_norm': 9.543654441833496, 'learning_rate': 5.306289454385379e-07, 'epoch': 0.32112828858150255, 'num_input_tokens_seen': 4160749568, 'completed': '53.81% (1_984 / 3_687)', 'remaining time': '15:07:27', 'throughput': '8267.03', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 01:59:17,385 >> {'loss': 0.4597, 'grad_norm': 15.082742691040039, 'learning_rate': 5.302256391067354e-07, 'epoch': 0.3213995117982099, 'num_input_tokens_seen': 4162846720, 'completed': '53.84% (1_985 / 3_687)', 'remaining time': '15:06:53', 'throughput': '8540.84', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 01:59:49,529 >> {'loss': 0.4349, 'grad_norm': 14.424219131469727, 'learning_rate': 5.29822348688586e-07, 'epoch': 0.3216707350149173, 'num_input_tokens_seen': 4164943872, 'completed': '53.86% (1_986 / 3_687)', 'remaining time': '15:06:21', 'throughput': '8155.49', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 02:00:22,894 >> {'loss': 0.4877, 'grad_norm': 17.661540985107422, 'learning_rate': 5.294190745086426e-07, 'epoch': 0.32194195823162464, 'num_input_tokens_seen': 4167041024, 'completed': '53.89% (1_987 / 3_687)', 'remaining time': '15:05:51', 'throughput': '7856.69', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 02:00:52,013 >> {'loss': 0.6855, 'grad_norm': 21.022674560546875, 'learning_rate': 5.290158168914447e-07, 'epoch': 0.322213181448332, 'num_input_tokens_seen': 4169138176, 'completed': '53.92% (1_988 / 3_687)', 'remaining time': '15:05:15', 'throughput': '9002.50', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 02:01:21,146 >> {'loss': 0.616, 'grad_norm': 19.004026412963867, 'learning_rate': 5.28612576161519e-07, 'epoch': 0.3224844046650393, 'num_input_tokens_seen': 4171235328, 'completed': '53.95% (1_989 / 3_687)', 'remaining time': '15:04:39', 'throughput': '8998.12', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 02:01:49,817 >> {'loss': 0.5411, 'grad_norm': 16.85509490966797, 'learning_rate': 5.282093526433781e-07, 'epoch': 0.32275562788174667, 'num_input_tokens_seen': 4173332480, 'completed': '53.97% (1_990 / 3_687)', 'remaining time': '15:04:02', 'throughput': '9143.47', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 02:02:19,876 >> {'loss': 0.2902, 'grad_norm': 14.019049644470215, 'learning_rate': 5.27806146661521e-07, 'epoch': 0.323026851098454, 'num_input_tokens_seen': 4175429632, 'completed': '54.00% (1_991 / 3_687)', 'remaining time': '15:03:28', 'throughput': '8720.96', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 02:02:51,169 >> {'loss': 0.3034, 'grad_norm': 16.84299087524414, 'learning_rate': 5.274029585404326e-07, 'epoch': 0.3232980743151614, 'num_input_tokens_seen': 4177526784, 'completed': '54.03% (1_992 / 3_687)', 'remaining time': '15:02:55', 'throughput': '8377.10', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 02:03:25,753 >> {'loss': 0.3721, 'grad_norm': 13.026562690734863, 'learning_rate': 5.269997886045833e-07, 'epoch': 0.32356929753186875, 'num_input_tokens_seen': 4179623936, 'completed': '54.05% (1_993 / 3_687)', 'remaining time': '15:02:27', 'throughput': '7579.81', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 02:03:55,120 >> {'loss': 0.5713, 'grad_norm': 16.872188568115234, 'learning_rate': 5.265966371784287e-07, 'epoch': 0.32384052074857606, 'num_input_tokens_seen': 4181721088, 'completed': '54.08% (1_994 / 3_687)', 'remaining time': '15:01:51', 'throughput': '8926.51', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 02:04:25,783 >> {'loss': 0.2909, 'grad_norm': 16.673782348632812, 'learning_rate': 5.261935045864101e-07, 'epoch': 0.3241117439652834, 'num_input_tokens_seen': 4183818240, 'completed': '54.11% (1_995 / 3_687)', 'remaining time': '15:01:17', 'throughput': '8549.05', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 02:04:57,508 >> {'loss': 0.4015, 'grad_norm': 14.819953918457031, 'learning_rate': 5.257903911529532e-07, 'epoch': 0.3243829671819908, 'num_input_tokens_seen': 4185915392, 'completed': '54.14% (1_996 / 3_687)', 'remaining time': '15:00:45', 'throughput': '8263.08', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 02:05:27,109 >> {'loss': 0.2481, 'grad_norm': 13.379781723022461, 'learning_rate': 5.253872972024681e-07, 'epoch': 0.32465419039869814, 'num_input_tokens_seen': 4188012544, 'completed': '54.16% (1_997 / 3_687)', 'remaining time': '15:00:10', 'throughput': '8856.08', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 02:05:55,192 >> {'loss': 0.376, 'grad_norm': 14.248574256896973, 'learning_rate': 5.249842230593497e-07, 'epoch': 0.3249254136154055, 'num_input_tokens_seen': 4190109696, 'completed': '54.19% (1_998 / 3_687)', 'remaining time': '14:59:32', 'throughput': '9334.35', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 02:06:26,095 >> {'loss': 0.3937, 'grad_norm': 22.307329177856445, 'learning_rate': 5.245811690479766e-07, 'epoch': 0.3251966368321128, 'num_input_tokens_seen': 4192206848, 'completed': '54.22% (1_999 / 3_687)', 'remaining time': '14:58:59', 'throughput': '8483.01', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 02:06:56,437 >> {'loss': 0.3849, 'grad_norm': 14.40904426574707, 'learning_rate': 5.241781354927113e-07, 'epoch': 0.32546786004882017, 'num_input_tokens_seen': 4194304000, 'completed': '54.24% (2_000 / 3_687)', 'remaining time': '14:58:24', 'throughput': '8639.61', 'gpu_mem_free': '5581MB'} +/scratch3/workspace/ctpham_umass_edu-ft/envs/prolong-final/lib/python3.10/site-packages/torch/distributed/fsdp/fully_sharded_data_parallel.py:689: FutureWarning: FSDP.state_dict_type() and FSDP.set_state_dict_type() are being deprecated. Please use APIs, get_state_dict() and set_state_dict(), which can support different parallelisms, FSDP1, FSDP2, DDP. API doc: https://pytorch.org/docs/stable/distributed.checkpoint.html#torch.distributed.checkpoint.state_dict.get_state_dict .Tutorial: https://pytorch.org/tutorials/recipes/distributed_checkpoint_recipe.html . + warnings.warn( +[INFO|trainer.py:3503] 2024-12-26 02:07:20,350 >> Saving model checkpoint to /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/checkpoint-2000 +[INFO|configuration_utils.py:472] 2024-12-26 02:07:20,353 >> Configuration saved in /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/checkpoint-2000/config.json +[INFO|configuration_utils.py:807] 2024-12-26 02:07:20,354 >> Configuration saved in /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/checkpoint-2000/generation_config.json +[INFO|modeling_utils.py:2807] 2024-12-26 02:08:16,686 >> The model is bigger than the maximum size per checkpoint (5GB) and is going to be split in 7 checkpoint shards. You can find where each parameters has been saved in the index located at /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/checkpoint-2000/model.safetensors.index.json. +[INFO|tokenization_utils_base.py:2684] 2024-12-26 02:08:16,690 >> tokenizer config file saved in /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/checkpoint-2000/tokenizer_config.json +[INFO|tokenization_utils_base.py:2693] 2024-12-26 02:08:16,690 >> Special tokens file saved in /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/checkpoint-2000/special_tokens_map.json +/scratch3/workspace/ctpham_umass_edu-ft/envs/prolong-final/lib/python3.10/site-packages/torch/distributed/fsdp/fully_sharded_data_parallel.py:689: FutureWarning: FSDP.state_dict_type() and FSDP.set_state_dict_type() are being deprecated. Please use APIs, get_state_dict() and set_state_dict(), which can support different parallelisms, FSDP1, FSDP2, DDP. API doc: https://pytorch.org/docs/stable/distributed.checkpoint.html#torch.distributed.checkpoint.state_dict.get_state_dict .Tutorial: https://pytorch.org/tutorials/recipes/distributed_checkpoint_recipe.html . + warnings.warn( +[WARNING|trainer.py:868] 2024-12-26 02:12:10,689 >> Save streaming dataset state: {'epoch': 0, 'sample_in_epoch': 4000, 'num_canonical_nodes': 1, 'shuffle_seed': 42, 'initial_physical_nodes': 1} +/scratch3/workspace/ctpham_umass_edu-ft/envs/prolong-final/lib/python3.10/site-packages/torch/utils/checkpoint.py:1399: FutureWarning: `torch.cpu.amp.autocast(args...)` is deprecated. Please use `torch.amp.autocast('cpu', args...)` instead. + with device_autocast_ctx, torch.cpu.amp.autocast(**cpu_autocast_kwargs), recompute_context: # type: ignore[attr-defined] +[INFO|trainer.py:175] 2024-12-26 02:12:42,740 >> {'loss': 0.3802, 'grad_norm': 18.996400833129883, 'learning_rate': 5.237751227179001e-07, 'epoch': 0.32573908326552753, 'num_input_tokens_seen': 4196401152, 'completed': '54.27% (2_001 / 3_687)', 'remaining time': '15:05:14', 'throughput': '756.98', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 02:13:13,335 >> {'loss': 0.5296, 'grad_norm': 17.524658203125, 'learning_rate': 5.233721310478722e-07, 'epoch': 0.3260103064822349, 'num_input_tokens_seen': 4198498304, 'completed': '54.30% (2_002 / 3_687)', 'remaining time': '15:04:39', 'throughput': '8568.32', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 02:13:43,119 >> {'loss': 0.2159, 'grad_norm': 11.911806106567383, 'learning_rate': 5.2296916080694e-07, 'epoch': 0.32628152969894225, 'num_input_tokens_seen': 4200595456, 'completed': '54.33% (2_003 / 3_687)', 'remaining time': '15:04:04', 'throughput': '8801.44', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 02:14:16,989 >> {'loss': 0.4125, 'grad_norm': 17.387310028076172, 'learning_rate': 5.225662123193985e-07, 'epoch': 0.32655275291564956, 'num_input_tokens_seen': 4202692608, 'completed': '54.35% (2_004 / 3_687)', 'remaining time': '15:03:34', 'throughput': '7739.64', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 02:14:47,142 >> {'loss': 0.3276, 'grad_norm': 17.1206111907959, 'learning_rate': 5.221632859095254e-07, 'epoch': 0.3268239761323569, 'num_input_tokens_seen': 4204789760, 'completed': '54.38% (2_005 / 3_687)', 'remaining time': '15:02:59', 'throughput': '8694.03', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 02:15:14,679 >> {'loss': 0.4545, 'grad_norm': 17.467138290405273, 'learning_rate': 5.217603819015805e-07, 'epoch': 0.3270951993490643, 'num_input_tokens_seen': 4206886912, 'completed': '54.41% (2_006 / 3_687)', 'remaining time': '15:02:20', 'throughput': '9519.54', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 02:15:46,881 >> {'loss': 0.7608, 'grad_norm': 22.18386459350586, 'learning_rate': 5.213575006198057e-07, 'epoch': 0.32736642256577164, 'num_input_tokens_seen': 4208984064, 'completed': '54.43% (2_007 / 3_687)', 'remaining time': '15:01:48', 'throughput': '8140.61', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 02:16:17,200 >> {'loss': 0.512, 'grad_norm': 18.72940444946289, 'learning_rate': 5.209546423884246e-07, 'epoch': 0.327637645782479, 'num_input_tokens_seen': 4211081216, 'completed': '54.46% (2_008 / 3_687)', 'remaining time': '15:01:13', 'throughput': '8646.07', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 02:16:48,696 >> {'loss': 0.3453, 'grad_norm': 15.777796745300293, 'learning_rate': 5.20551807531642e-07, 'epoch': 0.3279088689991863, 'num_input_tokens_seen': 4213178368, 'completed': '54.49% (2_009 / 3_687)', 'remaining time': '15:00:40', 'throughput': '8323.06', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 02:17:18,401 >> {'loss': 0.6098, 'grad_norm': 20.98493194580078, 'learning_rate': 5.20148996373644e-07, 'epoch': 0.32818009221589367, 'num_input_tokens_seen': 4215275520, 'completed': '54.52% (2_010 / 3_687)', 'remaining time': '15:00:04', 'throughput': '8824.94', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 02:17:48,462 >> {'loss': 0.3278, 'grad_norm': 13.770686149597168, 'learning_rate': 5.197462092385978e-07, 'epoch': 0.32845131543260103, 'num_input_tokens_seen': 4217372672, 'completed': '54.54% (2_011 / 3_687)', 'remaining time': '14:59:29', 'throughput': '8720.58', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 02:18:19,096 >> {'loss': 0.1932, 'grad_norm': 10.093859672546387, 'learning_rate': 5.193434464506511e-07, 'epoch': 0.3287225386493084, 'num_input_tokens_seen': 4219469824, 'completed': '54.57% (2_012 / 3_687)', 'remaining time': '14:58:55', 'throughput': '8557.32', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 02:18:48,920 >> {'loss': 0.3751, 'grad_norm': 16.35398292541504, 'learning_rate': 5.189407083339324e-07, 'epoch': 0.32899376186601575, 'num_input_tokens_seen': 4221566976, 'completed': '54.60% (2_013 / 3_687)', 'remaining time': '14:58:19', 'throughput': '8789.57', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 02:19:19,019 >> {'loss': 0.2063, 'grad_norm': 10.368598937988281, 'learning_rate': 5.185379952125494e-07, 'epoch': 0.32926498508272306, 'num_input_tokens_seen': 4223664128, 'completed': '54.62% (2_014 / 3_687)', 'remaining time': '14:57:44', 'throughput': '8709.29', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 02:19:49,808 >> {'loss': 0.3463, 'grad_norm': 15.229095458984375, 'learning_rate': 5.18135307410591e-07, 'epoch': 0.3295362082994304, 'num_input_tokens_seen': 4225761280, 'completed': '54.65% (2_015 / 3_687)', 'remaining time': '14:57:10', 'throughput': '8514.47', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 02:20:20,900 >> {'loss': 0.3193, 'grad_norm': 21.608442306518555, 'learning_rate': 5.177326452521242e-07, 'epoch': 0.3298074315161378, 'num_input_tokens_seen': 4227858432, 'completed': '54.68% (2_016 / 3_687)', 'remaining time': '14:56:36', 'throughput': '8431.05', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 02:20:51,508 >> {'loss': 0.3732, 'grad_norm': 16.298704147338867, 'learning_rate': 5.17330009061197e-07, 'epoch': 0.33007865473284514, 'num_input_tokens_seen': 4229955584, 'completed': '54.71% (2_017 / 3_687)', 'remaining time': '14:56:02', 'throughput': '8564.58', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 02:21:18,576 >> {'loss': 0.3864, 'grad_norm': 17.584896087646484, 'learning_rate': 5.169273991618351e-07, 'epoch': 0.3303498779495525, 'num_input_tokens_seen': 4232052736, 'completed': '54.73% (2_018 / 3_687)', 'remaining time': '14:55:23', 'throughput': '9684.54', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 02:21:52,464 >> {'loss': 0.3894, 'grad_norm': 17.949689865112305, 'learning_rate': 5.165248158780441e-07, 'epoch': 0.3306211011662598, 'num_input_tokens_seen': 4234149888, 'completed': '54.76% (2_019 / 3_687)', 'remaining time': '14:54:53', 'throughput': '7735.78', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 02:22:19,602 >> {'loss': 0.7639, 'grad_norm': 22.562408447265625, 'learning_rate': 5.161222595338073e-07, 'epoch': 0.3308923243829672, 'num_input_tokens_seen': 4236247040, 'completed': '54.79% (2_020 / 3_687)', 'remaining time': '14:54:14', 'throughput': '9659.51', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 02:22:52,612 >> {'loss': 0.3913, 'grad_norm': 15.829678535461426, 'learning_rate': 5.157197304530869e-07, 'epoch': 0.33116354759967453, 'num_input_tokens_seen': 4238344192, 'completed': '54.81% (2_021 / 3_687)', 'remaining time': '14:53:43', 'throughput': '7941.47', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 02:23:23,094 >> {'loss': 0.4058, 'grad_norm': 19.865497589111328, 'learning_rate': 5.15317228959823e-07, 'epoch': 0.3314347708163819, 'num_input_tokens_seen': 4240441344, 'completed': '54.84% (2_022 / 3_687)', 'remaining time': '14:53:08', 'throughput': '8599.91', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 02:23:53,062 >> {'loss': 0.61, 'grad_norm': 20.204696655273438, 'learning_rate': 5.149147553779335e-07, 'epoch': 0.33170599403308926, 'num_input_tokens_seen': 4242538496, 'completed': '54.87% (2_023 / 3_687)', 'remaining time': '14:52:33', 'throughput': '8747.51', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 02:24:27,007 >> {'loss': 0.2718, 'grad_norm': 11.350019454956055, 'learning_rate': 5.145123100313137e-07, 'epoch': 0.33197721724979656, 'num_input_tokens_seen': 4244635648, 'completed': '54.90% (2_024 / 3_687)', 'remaining time': '14:52:03', 'throughput': '7722.59', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 02:24:57,757 >> {'loss': 0.4187, 'grad_norm': 20.93108558654785, 'learning_rate': 5.141098932438365e-07, 'epoch': 0.3322484404665039, 'num_input_tokens_seen': 4246732800, 'completed': '54.92% (2_025 / 3_687)', 'remaining time': '14:51:29', 'throughput': '8524.80', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 02:25:33,948 >> {'loss': 0.3879, 'grad_norm': 17.941852569580078, 'learning_rate': 5.137075053393512e-07, 'epoch': 0.3325196636832113, 'num_input_tokens_seen': 4248829952, 'completed': '54.95% (2_026 / 3_687)', 'remaining time': '14:51:02', 'throughput': '7243.52', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 02:26:03,935 >> {'loss': 0.6891, 'grad_norm': 23.98504638671875, 'learning_rate': 5.133051466416849e-07, 'epoch': 0.33279088689991865, 'num_input_tokens_seen': 4250927104, 'completed': '54.98% (2_027 / 3_687)', 'remaining time': '14:50:27', 'throughput': '8741.87', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 02:26:34,294 >> {'loss': 0.3502, 'grad_norm': 17.1292667388916, 'learning_rate': 5.129028174746399e-07, 'epoch': 0.333062110116626, 'num_input_tokens_seen': 4253024256, 'completed': '55.00% (2_028 / 3_687)', 'remaining time': '14:49:52', 'throughput': '8634.74', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 02:27:03,481 >> {'loss': 0.939, 'grad_norm': 23.580167770385742, 'learning_rate': 5.12500518161996e-07, 'epoch': 0.3333333333333333, 'num_input_tokens_seen': 4255121408, 'completed': '55.03% (2_029 / 3_687)', 'remaining time': '14:49:16', 'throughput': '8981.50', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 02:27:36,513 >> {'loss': 0.3974, 'grad_norm': 16.317161560058594, 'learning_rate': 5.120982490275081e-07, 'epoch': 0.3336045565500407, 'num_input_tokens_seen': 4257218560, 'completed': '55.06% (2_030 / 3_687)', 'remaining time': '14:48:45', 'throughput': '7936.15', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 02:28:05,360 >> {'loss': 0.7093, 'grad_norm': 19.563129425048828, 'learning_rate': 5.116960103949071e-07, 'epoch': 0.33387577976674804, 'num_input_tokens_seen': 4259315712, 'completed': '55.09% (2_031 / 3_687)', 'remaining time': '14:48:08', 'throughput': '9087.33', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 02:28:37,821 >> {'loss': 0.3047, 'grad_norm': 14.194764137268066, 'learning_rate': 5.112938025878994e-07, 'epoch': 0.3341470029834554, 'num_input_tokens_seen': 4261412864, 'completed': '55.11% (2_032 / 3_687)', 'remaining time': '14:47:37', 'throughput': '8075.67', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 02:29:07,036 >> {'loss': 0.2439, 'grad_norm': 12.254755973815918, 'learning_rate': 5.108916259301663e-07, 'epoch': 0.33441822620016276, 'num_input_tokens_seen': 4263510016, 'completed': '55.14% (2_033 / 3_687)', 'remaining time': '14:47:00', 'throughput': '8972.83', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 02:29:40,702 >> {'loss': 0.3246, 'grad_norm': 13.140419006347656, 'learning_rate': 5.104894807453647e-07, 'epoch': 0.33468944941687007, 'num_input_tokens_seen': 4265607168, 'completed': '55.17% (2_034 / 3_687)', 'remaining time': '14:46:30', 'throughput': '7786.67', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 02:30:10,034 >> {'loss': 0.5276, 'grad_norm': 18.896413803100586, 'learning_rate': 5.100873673571253e-07, 'epoch': 0.3349606726335774, 'num_input_tokens_seen': 4267704320, 'completed': '55.19% (2_035 / 3_687)', 'remaining time': '14:45:54', 'throughput': '8937.24', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 02:30:43,945 >> {'loss': 0.335, 'grad_norm': 13.808533668518066, 'learning_rate': 5.096852860890541e-07, 'epoch': 0.3352318958502848, 'num_input_tokens_seen': 4269801472, 'completed': '55.22% (2_036 / 3_687)', 'remaining time': '14:45:24', 'throughput': '7730.30', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 02:31:13,083 >> {'loss': 0.6184, 'grad_norm': 21.51602554321289, 'learning_rate': 5.092832372647304e-07, 'epoch': 0.33550311906699215, 'num_input_tokens_seen': 4271898624, 'completed': '55.25% (2_037 / 3_687)', 'remaining time': '14:44:48', 'throughput': '8996.57', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 02:31:45,133 >> {'loss': 0.519, 'grad_norm': 18.561599731445312, 'learning_rate': 5.08881221207708e-07, 'epoch': 0.3357743422836995, 'num_input_tokens_seen': 4273995776, 'completed': '55.28% (2_038 / 3_687)', 'remaining time': '14:44:16', 'throughput': '8179.35', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 02:32:16,343 >> {'loss': 0.2667, 'grad_norm': 15.325385093688965, 'learning_rate': 5.084792382415141e-07, 'epoch': 0.3360455655004068, 'num_input_tokens_seen': 4276092928, 'completed': '55.30% (2_039 / 3_687)', 'remaining time': '14:43:42', 'throughput': '8399.28', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 02:32:45,382 >> {'loss': 0.429, 'grad_norm': 15.208858489990234, 'learning_rate': 5.080772886896493e-07, 'epoch': 0.3363167887171142, 'num_input_tokens_seen': 4278190080, 'completed': '55.33% (2_040 / 3_687)', 'remaining time': '14:43:06', 'throughput': '9027.18', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 02:33:15,269 >> {'loss': 0.2648, 'grad_norm': 13.985490798950195, 'learning_rate': 5.076753728755871e-07, 'epoch': 0.33658801193382154, 'num_input_tokens_seen': 4280287232, 'completed': '55.36% (2_041 / 3_687)', 'remaining time': '14:42:31', 'throughput': '8771.32', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 02:33:45,371 >> {'loss': 0.3776, 'grad_norm': 19.45819664001465, 'learning_rate': 5.072734911227746e-07, 'epoch': 0.3368592351505289, 'num_input_tokens_seen': 4282384384, 'completed': '55.38% (2_042 / 3_687)', 'remaining time': '14:41:56', 'throughput': '8708.56', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 02:34:15,996 >> {'loss': 0.1879, 'grad_norm': 11.178092956542969, 'learning_rate': 5.068716437546305e-07, 'epoch': 0.33713045836723626, 'num_input_tokens_seen': 4284481536, 'completed': '55.41% (2_043 / 3_687)', 'remaining time': '14:41:22', 'throughput': '8559.84', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 02:34:49,613 >> {'loss': 0.5829, 'grad_norm': 19.116785049438477, 'learning_rate': 5.06469831094546e-07, 'epoch': 0.33740168158394357, 'num_input_tokens_seen': 4286578688, 'completed': '55.44% (2_044 / 3_687)', 'remaining time': '14:40:52', 'throughput': '7797.87', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 02:35:18,656 >> {'loss': 0.4346, 'grad_norm': 17.29070281982422, 'learning_rate': 5.060680534658852e-07, 'epoch': 0.33767290480065093, 'num_input_tokens_seen': 4288675840, 'completed': '55.47% (2_045 / 3_687)', 'remaining time': '14:40:15', 'throughput': '9025.96', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 02:35:48,760 >> {'loss': 0.3758, 'grad_norm': 11.40611457824707, 'learning_rate': 5.056663111919833e-07, 'epoch': 0.3379441280173583, 'num_input_tokens_seen': 4290772992, 'completed': '55.49% (2_046 / 3_687)', 'remaining time': '14:39:40', 'throughput': '8707.91', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 02:36:17,726 >> {'loss': 0.2305, 'grad_norm': 10.537538528442383, 'learning_rate': 5.052646045961471e-07, 'epoch': 0.33821535123406565, 'num_input_tokens_seen': 4292870144, 'completed': '55.52% (2_047 / 3_687)', 'remaining time': '14:39:04', 'throughput': '9050.24', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 02:36:46,914 >> {'loss': 0.3792, 'grad_norm': 15.972824096679688, 'learning_rate': 5.048629340016548e-07, 'epoch': 0.33848657445077296, 'num_input_tokens_seen': 4294967296, 'completed': '55.55% (2_048 / 3_687)', 'remaining time': '14:38:28', 'throughput': '8981.31', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 02:37:17,144 >> {'loss': 0.3346, 'grad_norm': 13.639211654663086, 'learning_rate': 5.044612997317557e-07, 'epoch': 0.3387577976674803, 'num_input_tokens_seen': 4297064448, 'completed': '55.57% (2_049 / 3_687)', 'remaining time': '14:37:53', 'throughput': '8671.52', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 02:37:48,662 >> {'loss': 0.6396, 'grad_norm': 17.853118896484375, 'learning_rate': 5.040597021096695e-07, 'epoch': 0.3390290208841877, 'num_input_tokens_seen': 4299161600, 'completed': '55.60% (2_050 / 3_687)', 'remaining time': '14:37:20', 'throughput': '8317.17', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 02:38:22,120 >> {'loss': 0.4423, 'grad_norm': 18.226207733154297, 'learning_rate': 5.03658141458587e-07, 'epoch': 0.33930024410089504, 'num_input_tokens_seen': 4301258752, 'completed': '55.63% (2_051 / 3_687)', 'remaining time': '14:36:50', 'throughput': '7835.11', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 02:38:54,167 >> {'loss': 0.3318, 'grad_norm': 14.695561408996582, 'learning_rate': 5.032566181016688e-07, 'epoch': 0.3395714673176024, 'num_input_tokens_seen': 4303355904, 'completed': '55.66% (2_052 / 3_687)', 'remaining time': '14:36:17', 'throughput': '8180.00', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 02:39:21,555 >> {'loss': 0.6093, 'grad_norm': 18.532405853271484, 'learning_rate': 5.028551323620458e-07, 'epoch': 0.3398426905343097, 'num_input_tokens_seen': 4305453056, 'completed': '55.68% (2_053 / 3_687)', 'remaining time': '14:35:39', 'throughput': '9571.58', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 02:39:51,695 >> {'loss': 0.4432, 'grad_norm': 15.558454513549805, 'learning_rate': 5.024536845628181e-07, 'epoch': 0.34011391375101707, 'num_input_tokens_seen': 4307550208, 'completed': '55.71% (2_054 / 3_687)', 'remaining time': '14:35:04', 'throughput': '8697.56', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 02:40:22,837 >> {'loss': 0.6549, 'grad_norm': 22.573936462402344, 'learning_rate': 5.020522750270558e-07, 'epoch': 0.34038513696772443, 'num_input_tokens_seen': 4309647360, 'completed': '55.74% (2_055 / 3_687)', 'remaining time': '14:34:31', 'throughput': '8417.65', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 02:40:52,720 >> {'loss': 0.27, 'grad_norm': 13.960949897766113, 'learning_rate': 5.01650904077798e-07, 'epoch': 0.3406563601844318, 'num_input_tokens_seen': 4311744512, 'completed': '55.76% (2_056 / 3_687)', 'remaining time': '14:33:56', 'throughput': '8772.35', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 02:41:22,211 >> {'loss': 0.6896, 'grad_norm': 19.82161521911621, 'learning_rate': 5.012495720380527e-07, 'epoch': 0.34092758340113916, 'num_input_tokens_seen': 4313841664, 'completed': '55.79% (2_057 / 3_687)', 'remaining time': '14:33:20', 'throughput': '8888.86', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 02:41:54,000 >> {'loss': 0.3401, 'grad_norm': 13.285265922546387, 'learning_rate': 5.008482792307968e-07, 'epoch': 0.34119880661784646, 'num_input_tokens_seen': 4315938816, 'completed': '55.82% (2_058 / 3_687)', 'remaining time': '14:32:48', 'throughput': '8246.62', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 02:42:26,964 >> {'loss': 0.3066, 'grad_norm': 12.160155296325684, 'learning_rate': 5.004470259789757e-07, 'epoch': 0.3414700298345538, 'num_input_tokens_seen': 4318035968, 'completed': '55.84% (2_059 / 3_687)', 'remaining time': '14:32:16', 'throughput': '7952.41', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 02:43:00,311 >> {'loss': 0.324, 'grad_norm': 12.414044380187988, 'learning_rate': 5.000458126055021e-07, 'epoch': 0.3417412530512612, 'num_input_tokens_seen': 4320133120, 'completed': '55.87% (2_060 / 3_687)', 'remaining time': '14:31:46', 'throughput': '7860.81', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 02:43:31,722 >> {'loss': 0.5181, 'grad_norm': 18.17831802368164, 'learning_rate': 4.996446394332578e-07, 'epoch': 0.34201247626796855, 'num_input_tokens_seen': 4322230272, 'completed': '55.90% (2_061 / 3_687)', 'remaining time': '14:31:13', 'throughput': '8345.69', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 02:44:04,322 >> {'loss': 0.3709, 'grad_norm': 14.670184135437012, 'learning_rate': 4.992435067850914e-07, 'epoch': 0.3422836994846759, 'num_input_tokens_seen': 4324327424, 'completed': '55.93% (2_062 / 3_687)', 'remaining time': '14:30:41', 'throughput': '8041.22', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 02:44:34,106 >> {'loss': 0.5221, 'grad_norm': 17.332820892333984, 'learning_rate': 4.988424149838192e-07, 'epoch': 0.3425549227013832, 'num_input_tokens_seen': 4326424576, 'completed': '55.95% (2_063 / 3_687)', 'remaining time': '14:30:06', 'throughput': '8801.63', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 02:45:04,121 >> {'loss': 0.7608, 'grad_norm': 22.735279083251953, 'learning_rate': 4.984413643522251e-07, 'epoch': 0.3428261459180906, 'num_input_tokens_seen': 4328521728, 'completed': '55.98% (2_064 / 3_687)', 'remaining time': '14:29:31', 'throughput': '8733.60', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 02:45:32,316 >> {'loss': 0.3522, 'grad_norm': 14.321521759033203, 'learning_rate': 4.980403552130589e-07, 'epoch': 0.34309736913479794, 'num_input_tokens_seen': 4330618880, 'completed': '56.01% (2_065 / 3_687)', 'remaining time': '14:28:54', 'throughput': '9297.54', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 02:46:01,482 >> {'loss': 0.4359, 'grad_norm': 16.680944442749023, 'learning_rate': 4.976393878890375e-07, 'epoch': 0.3433685923515053, 'num_input_tokens_seen': 4332716032, 'completed': '56.03% (2_066 / 3_687)', 'remaining time': '14:28:18', 'throughput': '8988.10', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 02:46:31,498 >> {'loss': 0.4501, 'grad_norm': 15.195837020874023, 'learning_rate': 4.972384627028442e-07, 'epoch': 0.34363981556821266, 'num_input_tokens_seen': 4334813184, 'completed': '56.06% (2_067 / 3_687)', 'remaining time': '14:27:43', 'throughput': '8733.56', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 02:47:04,062 >> {'loss': 0.3573, 'grad_norm': 13.431507110595703, 'learning_rate': 4.968375799771285e-07, 'epoch': 0.34391103878491996, 'num_input_tokens_seen': 4336910336, 'completed': '56.09% (2_068 / 3_687)', 'remaining time': '14:27:11', 'throughput': '8049.97', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 02:47:33,016 >> {'loss': 0.3762, 'grad_norm': 12.854487419128418, 'learning_rate': 4.964367400345053e-07, 'epoch': 0.3441822620016273, 'num_input_tokens_seen': 4339007488, 'completed': '56.12% (2_069 / 3_687)', 'remaining time': '14:26:35', 'throughput': '9053.97', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 02:48:02,436 >> {'loss': 0.7081, 'grad_norm': 17.647762298583984, 'learning_rate': 4.960359431975555e-07, 'epoch': 0.3444534852183347, 'num_input_tokens_seen': 4341104640, 'completed': '56.14% (2_070 / 3_687)', 'remaining time': '14:26:00', 'throughput': '8910.30', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 02:48:33,412 >> {'loss': 0.7699, 'grad_norm': 21.555879592895508, 'learning_rate': 4.956351897888256e-07, 'epoch': 0.34472470843504205, 'num_input_tokens_seen': 4343201792, 'completed': '56.17% (2_071 / 3_687)', 'remaining time': '14:25:26', 'throughput': '8462.71', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 02:49:01,236 >> {'loss': 0.5833, 'grad_norm': 17.82149887084961, 'learning_rate': 4.952344801308258e-07, 'epoch': 0.3449959316517494, 'num_input_tokens_seen': 4345298944, 'completed': '56.20% (2_072 / 3_687)', 'remaining time': '14:24:48', 'throughput': '9421.71', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 02:49:29,202 >> {'loss': 0.3707, 'grad_norm': 13.02746295928955, 'learning_rate': 4.948338145460324e-07, 'epoch': 0.3452671548684567, 'num_input_tokens_seen': 4347396096, 'completed': '56.22% (2_073 / 3_687)', 'remaining time': '14:24:11', 'throughput': '9373.66', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 02:49:59,422 >> {'loss': 0.6831, 'grad_norm': 18.583114624023438, 'learning_rate': 4.944331933568858e-07, 'epoch': 0.3455383780851641, 'num_input_tokens_seen': 4349493248, 'completed': '56.25% (2_074 / 3_687)', 'remaining time': '14:23:37', 'throughput': '8674.41', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 02:50:25,829 >> {'loss': 0.618, 'grad_norm': 18.889436721801758, 'learning_rate': 4.94032616885791e-07, 'epoch': 0.34580960130187144, 'num_input_tokens_seen': 4351590400, 'completed': '56.28% (2_075 / 3_687)', 'remaining time': '14:22:57', 'throughput': '9927.23', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 02:50:59,635 >> {'loss': 0.4065, 'grad_norm': 14.465885162353516, 'learning_rate': 4.936320854551163e-07, 'epoch': 0.3460808245185788, 'num_input_tokens_seen': 4353687552, 'completed': '56.31% (2_076 / 3_687)', 'remaining time': '14:22:27', 'throughput': '7754.26', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 02:51:30,948 >> {'loss': 0.2518, 'grad_norm': 11.185941696166992, 'learning_rate': 4.932315993871942e-07, 'epoch': 0.34635204773528616, 'num_input_tokens_seen': 4355784704, 'completed': '56.33% (2_077 / 3_687)', 'remaining time': '14:21:54', 'throughput': '8371.70', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 02:52:02,946 >> {'loss': 0.3067, 'grad_norm': 14.272321701049805, 'learning_rate': 4.928311590043206e-07, 'epoch': 0.34662327095199347, 'num_input_tokens_seen': 4357881856, 'completed': '56.36% (2_078 / 3_687)', 'remaining time': '14:21:22', 'throughput': '8192.55', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 02:52:33,017 >> {'loss': 0.6866, 'grad_norm': 23.103809356689453, 'learning_rate': 4.924307646287549e-07, 'epoch': 0.34689449416870083, 'num_input_tokens_seen': 4359979008, 'completed': '56.39% (2_079 / 3_687)', 'remaining time': '14:20:47', 'throughput': '8717.66', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 02:53:00,405 >> {'loss': 0.7426, 'grad_norm': 24.843984603881836, 'learning_rate': 4.920304165827192e-07, 'epoch': 0.3471657173854082, 'num_input_tokens_seen': 4362076160, 'completed': '56.41% (2_080 / 3_687)', 'remaining time': '14:20:09', 'throughput': '9571.46', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 02:53:30,433 >> {'loss': 0.4618, 'grad_norm': 15.894787788391113, 'learning_rate': 4.916301151883983e-07, 'epoch': 0.34743694060211555, 'num_input_tokens_seen': 4364173312, 'completed': '56.44% (2_081 / 3_687)', 'remaining time': '14:19:34', 'throughput': '8729.84', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 02:54:00,534 >> {'loss': 0.6288, 'grad_norm': 19.025161743164062, 'learning_rate': 4.912298607679396e-07, 'epoch': 0.3477081638188229, 'num_input_tokens_seen': 4366270464, 'completed': '56.47% (2_082 / 3_687)', 'remaining time': '14:19:00', 'throughput': '8708.98', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 02:54:29,404 >> {'loss': 0.4496, 'grad_norm': 17.685871124267578, 'learning_rate': 4.908296536434525e-07, 'epoch': 0.3479793870355302, 'num_input_tokens_seen': 4368367616, 'completed': '56.50% (2_083 / 3_687)', 'remaining time': '14:18:23', 'throughput': '9080.17', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 02:54:59,702 >> {'loss': 0.7546, 'grad_norm': 26.235916137695312, 'learning_rate': 4.904294941370086e-07, 'epoch': 0.3482506102522376, 'num_input_tokens_seen': 4370464768, 'completed': '56.52% (2_084 / 3_687)', 'remaining time': '14:17:49', 'throughput': '8651.95', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 02:55:29,186 >> {'loss': 0.2447, 'grad_norm': 12.8169584274292, 'learning_rate': 4.900293825706408e-07, 'epoch': 0.34852183346894494, 'num_input_tokens_seen': 4372561920, 'completed': '56.55% (2_085 / 3_687)', 'remaining time': '14:17:14', 'throughput': '8891.22', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 02:55:59,472 >> {'loss': 0.4664, 'grad_norm': 16.72005271911621, 'learning_rate': 4.89629319266344e-07, 'epoch': 0.3487930566856523, 'num_input_tokens_seen': 4374659072, 'completed': '56.58% (2_086 / 3_687)', 'remaining time': '14:16:39', 'throughput': '8655.49', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 02:56:32,021 >> {'loss': 0.3813, 'grad_norm': 14.927910804748535, 'learning_rate': 4.892293045460738e-07, 'epoch': 0.34906427990235966, 'num_input_tokens_seen': 4376756224, 'completed': '56.60% (2_087 / 3_687)', 'remaining time': '14:16:08', 'throughput': '8053.98', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 02:57:03,647 >> {'loss': 0.3661, 'grad_norm': 14.392501831054688, 'learning_rate': 4.88829338731747e-07, 'epoch': 0.34933550311906697, 'num_input_tokens_seen': 4378853376, 'completed': '56.63% (2_088 / 3_687)', 'remaining time': '14:15:35', 'throughput': '8288.86', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 02:57:33,723 >> {'loss': 0.3709, 'grad_norm': 15.27480697631836, 'learning_rate': 4.884294221452405e-07, 'epoch': 0.34960672633577433, 'num_input_tokens_seen': 4380950528, 'completed': '56.66% (2_089 / 3_687)', 'remaining time': '14:15:00', 'throughput': '8715.98', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 02:58:02,715 >> {'loss': 0.58, 'grad_norm': 16.14817237854004, 'learning_rate': 4.88029555108392e-07, 'epoch': 0.3498779495524817, 'num_input_tokens_seen': 4383047680, 'completed': '56.69% (2_090 / 3_687)', 'remaining time': '14:14:25', 'throughput': '9042.06', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 02:58:32,984 >> {'loss': 0.2754, 'grad_norm': 11.918871879577637, 'learning_rate': 4.876297379429995e-07, 'epoch': 0.35014917276918905, 'num_input_tokens_seen': 4385144832, 'completed': '56.71% (2_091 / 3_687)', 'remaining time': '14:13:50', 'throughput': '8660.42', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 02:59:02,391 >> {'loss': 0.453, 'grad_norm': 24.65549659729004, 'learning_rate': 4.872299709708206e-07, 'epoch': 0.3504203959858964, 'num_input_tokens_seen': 4387241984, 'completed': '56.74% (2_092 / 3_687)', 'remaining time': '14:13:15', 'throughput': '8914.28', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 02:59:36,672 >> {'loss': 0.3193, 'grad_norm': 13.130111694335938, 'learning_rate': 4.868302545135725e-07, 'epoch': 0.3506916192026037, 'num_input_tokens_seen': 4389339136, 'completed': '56.77% (2_093 / 3_687)', 'remaining time': '14:12:45', 'throughput': '7646.93', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 03:00:02,128 >> {'loss': 0.9149, 'grad_norm': 22.114221572875977, 'learning_rate': 4.864305888929318e-07, 'epoch': 0.3509628424193111, 'num_input_tokens_seen': 4391436288, 'completed': '56.79% (2_094 / 3_687)', 'remaining time': '14:12:05', 'throughput': '10297.88', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 03:00:31,615 >> {'loss': 0.7968, 'grad_norm': 22.851913452148438, 'learning_rate': 4.860309744305339e-07, 'epoch': 0.35123406563601844, 'num_input_tokens_seen': 4393533440, 'completed': '56.82% (2_095 / 3_687)', 'remaining time': '14:11:30', 'throughput': '8890.13', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 03:01:00,965 >> {'loss': 0.7142, 'grad_norm': 19.785030364990234, 'learning_rate': 4.856314114479738e-07, 'epoch': 0.3515052888527258, 'num_input_tokens_seen': 4395630592, 'completed': '56.85% (2_096 / 3_687)', 'remaining time': '14:10:54', 'throughput': '8931.84', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 03:01:32,028 >> {'loss': 0.3404, 'grad_norm': 13.180719375610352, 'learning_rate': 4.852319002668044e-07, 'epoch': 0.35177651206943317, 'num_input_tokens_seen': 4397727744, 'completed': '56.88% (2_097 / 3_687)', 'remaining time': '14:10:21', 'throughput': '8438.97', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 03:02:01,065 >> {'loss': 0.5252, 'grad_norm': 18.110231399536133, 'learning_rate': 4.848324412085367e-07, 'epoch': 0.3520477352861405, 'num_input_tokens_seen': 4399824896, 'completed': '56.90% (2_098 / 3_687)', 'remaining time': '14:09:45', 'throughput': '9028.02', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 03:02:30,035 >> {'loss': 0.4975, 'grad_norm': 18.176883697509766, 'learning_rate': 4.84433034594641e-07, 'epoch': 0.35231895850284783, 'num_input_tokens_seen': 4401922048, 'completed': '56.93% (2_099 / 3_687)', 'remaining time': '14:09:09', 'throughput': '9048.77', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 03:03:00,315 >> {'loss': 0.4609, 'grad_norm': 16.192501068115234, 'learning_rate': 4.840336807465439e-07, 'epoch': 0.3525901817195552, 'num_input_tokens_seen': 4404019200, 'completed': '56.96% (2_100 / 3_687)', 'remaining time': '14:08:35', 'throughput': '8657.41', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 03:03:29,913 >> {'loss': 0.4662, 'grad_norm': 15.730474472045898, 'learning_rate': 4.836343799856302e-07, 'epoch': 0.35286140493626256, 'num_input_tokens_seen': 4406116352, 'completed': '56.98% (2_101 / 3_687)', 'remaining time': '14:08:00', 'throughput': '8856.80', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 03:04:00,522 >> {'loss': 0.2086, 'grad_norm': 9.630202293395996, 'learning_rate': 4.83235132633242e-07, 'epoch': 0.3531326281529699, 'num_input_tokens_seen': 4408213504, 'completed': '57.01% (2_102 / 3_687)', 'remaining time': '14:07:26', 'throughput': '8564.56', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 03:04:31,386 >> {'loss': 0.2325, 'grad_norm': 10.681892395019531, 'learning_rate': 4.828359390106786e-07, 'epoch': 0.3534038513696772, 'num_input_tokens_seen': 4410310656, 'completed': '57.04% (2_103 / 3_687)', 'remaining time': '14:06:52', 'throughput': '8493.25', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 03:05:00,066 >> {'loss': 0.4176, 'grad_norm': 15.098684310913086, 'learning_rate': 4.824367994391958e-07, 'epoch': 0.3536750745863846, 'num_input_tokens_seen': 4412407808, 'completed': '57.07% (2_104 / 3_687)', 'remaining time': '14:06:16', 'throughput': '9140.09', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 03:05:32,104 >> {'loss': 0.2685, 'grad_norm': 11.902873039245605, 'learning_rate': 4.820377142400058e-07, 'epoch': 0.35394629780309195, 'num_input_tokens_seen': 4414504960, 'completed': '57.09% (2_105 / 3_687)', 'remaining time': '14:05:44', 'throughput': '8182.51', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 03:06:02,557 >> {'loss': 0.5198, 'grad_norm': 19.398834228515625, 'learning_rate': 4.816386837342771e-07, 'epoch': 0.3542175210197993, 'num_input_tokens_seen': 4416602112, 'completed': '57.12% (2_106 / 3_687)', 'remaining time': '14:05:10', 'throughput': '8608.04', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 03:06:34,621 >> {'loss': 0.5469, 'grad_norm': 21.086822509765625, 'learning_rate': 4.812397082431345e-07, 'epoch': 0.35448874423650667, 'num_input_tokens_seen': 4418699264, 'completed': '57.15% (2_107 / 3_687)', 'remaining time': '14:04:38', 'throughput': '8175.67', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 03:07:05,458 >> {'loss': 0.6044, 'grad_norm': 21.071117401123047, 'learning_rate': 4.808407880876581e-07, 'epoch': 0.354759967453214, 'num_input_tokens_seen': 4420796416, 'completed': '57.17% (2_108 / 3_687)', 'remaining time': '14:04:04', 'throughput': '8500.88', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 03:07:36,788 >> {'loss': 0.5796, 'grad_norm': 17.908489227294922, 'learning_rate': 4.804419235888838e-07, 'epoch': 0.35503119066992134, 'num_input_tokens_seen': 4422893568, 'completed': '57.20% (2_109 / 3_687)', 'remaining time': '14:03:31', 'throughput': '8367.30', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 03:08:09,238 >> {'loss': 0.4045, 'grad_norm': 15.03857135772705, 'learning_rate': 4.800431150678026e-07, 'epoch': 0.3553024138866287, 'num_input_tokens_seen': 4424990720, 'completed': '57.23% (2_110 / 3_687)', 'remaining time': '14:03:00', 'throughput': '8078.28', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 03:08:38,195 >> {'loss': 0.6847, 'grad_norm': 22.05478858947754, 'learning_rate': 4.796443628453603e-07, 'epoch': 0.35557363710333606, 'num_input_tokens_seen': 4427087872, 'completed': '57.26% (2_111 / 3_687)', 'remaining time': '14:02:24', 'throughput': '9052.98', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 03:09:05,421 >> {'loss': 0.5185, 'grad_norm': 18.630102157592773, 'learning_rate': 4.792456672424574e-07, 'epoch': 0.3558448603200434, 'num_input_tokens_seen': 4429185024, 'completed': '57.28% (2_112 / 3_687)', 'remaining time': '14:01:46', 'throughput': '9628.31', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 03:09:38,348 >> {'loss': 0.2514, 'grad_norm': 10.600038528442383, 'learning_rate': 4.788470285799492e-07, 'epoch': 0.3561160835367507, 'num_input_tokens_seen': 4431282176, 'completed': '57.31% (2_113 / 3_687)', 'remaining time': '14:01:15', 'throughput': '7961.50', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 03:10:08,581 >> {'loss': 0.3868, 'grad_norm': 15.612627029418945, 'learning_rate': 4.784484471786447e-07, 'epoch': 0.3563873067534581, 'num_input_tokens_seen': 4433379328, 'completed': '57.34% (2_114 / 3_687)', 'remaining time': '14:00:41', 'throughput': '8670.72', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 03:10:38,651 >> {'loss': 0.5039, 'grad_norm': 17.270404815673828, 'learning_rate': 4.780499233593073e-07, 'epoch': 0.35665852997016545, 'num_input_tokens_seen': 4435476480, 'completed': '57.36% (2_115 / 3_687)', 'remaining time': '14:00:06', 'throughput': '8717.86', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 03:11:11,776 >> {'loss': 0.4082, 'grad_norm': 17.62727928161621, 'learning_rate': 4.776514574426538e-07, 'epoch': 0.3569297531868728, 'num_input_tokens_seen': 4437573632, 'completed': '57.39% (2_116 / 3_687)', 'remaining time': '13:59:36', 'throughput': '7913.73', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 03:11:43,054 >> {'loss': 0.3864, 'grad_norm': 15.866716384887695, 'learning_rate': 4.772530497493541e-07, 'epoch': 0.3572009764035802, 'num_input_tokens_seen': 4439670784, 'completed': '57.42% (2_117 / 3_687)', 'remaining time': '13:59:03', 'throughput': '8381.00', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 03:12:13,103 >> {'loss': 0.505, 'grad_norm': 17.658111572265625, 'learning_rate': 4.768547006000317e-07, 'epoch': 0.3574721996202875, 'num_input_tokens_seen': 4441767936, 'completed': '57.45% (2_118 / 3_687)', 'remaining time': '13:58:28', 'throughput': '8723.84', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 03:12:43,474 >> {'loss': 0.5577, 'grad_norm': 17.13514518737793, 'learning_rate': 4.76456410315263e-07, 'epoch': 0.35774342283699484, 'num_input_tokens_seen': 4443865088, 'completed': '57.47% (2_119 / 3_687)', 'remaining time': '13:57:54', 'throughput': '8631.43', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 03:13:14,202 >> {'loss': 0.3055, 'grad_norm': 12.680811882019043, 'learning_rate': 4.7605817921557666e-07, 'epoch': 0.3580146460537022, 'num_input_tokens_seen': 4445962240, 'completed': '57.50% (2_120 / 3_687)', 'remaining time': '13:57:20', 'throughput': '8531.15', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 03:13:44,833 >> {'loss': 0.3165, 'grad_norm': 10.903252601623535, 'learning_rate': 4.756600076214541e-07, 'epoch': 0.35828586927040956, 'num_input_tokens_seen': 4448059392, 'completed': '57.53% (2_121 / 3_687)', 'remaining time': '13:56:47', 'throughput': '8558.13', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 03:14:12,025 >> {'loss': 0.489, 'grad_norm': 20.912986755371094, 'learning_rate': 4.7526189585332855e-07, 'epoch': 0.3585570924871169, 'num_input_tokens_seen': 4450156544, 'completed': '57.55% (2_122 / 3_687)', 'remaining time': '13:56:09', 'throughput': '9640.57', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 03:14:43,057 >> {'loss': 0.7097, 'grad_norm': 19.84836196899414, 'learning_rate': 4.748638442315851e-07, 'epoch': 0.35882831570382423, 'num_input_tokens_seen': 4452253696, 'completed': '57.58% (2_123 / 3_687)', 'remaining time': '13:55:35', 'throughput': '8447.63', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 03:15:14,427 >> {'loss': 0.4103, 'grad_norm': 14.485566139221191, 'learning_rate': 4.7446585307656074e-07, 'epoch': 0.3590995389205316, 'num_input_tokens_seen': 4454350848, 'completed': '57.61% (2_124 / 3_687)', 'remaining time': '13:55:03', 'throughput': '8356.47', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 03:15:47,712 >> {'loss': 0.6003, 'grad_norm': 17.28524398803711, 'learning_rate': 4.740679227085436e-07, 'epoch': 0.35937076213723895, 'num_input_tokens_seen': 4456448000, 'completed': '57.63% (2_125 / 3_687)', 'remaining time': '13:54:32', 'throughput': '7875.76', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 03:16:20,128 >> {'loss': 0.4924, 'grad_norm': 17.543121337890625, 'learning_rate': 4.7367005344777255e-07, 'epoch': 0.3596419853539463, 'num_input_tokens_seen': 4458545152, 'completed': '57.66% (2_126 / 3_687)', 'remaining time': '13:54:00', 'throughput': '8086.66', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 03:16:49,776 >> {'loss': 0.4224, 'grad_norm': 15.53244400024414, 'learning_rate': 4.732722456144381e-07, 'epoch': 0.3599132085706537, 'num_input_tokens_seen': 4460642304, 'completed': '57.69% (2_127 / 3_687)', 'remaining time': '13:53:25', 'throughput': '8841.98', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 03:17:22,269 >> {'loss': 0.4343, 'grad_norm': 15.329564094543457, 'learning_rate': 4.7287449952868084e-07, 'epoch': 0.360184431787361, 'num_input_tokens_seen': 4462739456, 'completed': '57.72% (2_128 / 3_687)', 'remaining time': '13:52:54', 'throughput': '8067.84', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 03:17:52,480 >> {'loss': 0.2533, 'grad_norm': 11.179370880126953, 'learning_rate': 4.724768155105914e-07, 'epoch': 0.36045565500406834, 'num_input_tokens_seen': 4464836608, 'completed': '57.74% (2_129 / 3_687)', 'remaining time': '13:52:20', 'throughput': '8677.12', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 03:18:22,042 >> {'loss': 0.6616, 'grad_norm': 19.77837371826172, 'learning_rate': 4.7207919388021076e-07, 'epoch': 0.3607268782207757, 'num_input_tokens_seen': 4466933760, 'completed': '57.77% (2_130 / 3_687)', 'remaining time': '13:51:45', 'throughput': '8867.48', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 03:18:53,908 >> {'loss': 0.4892, 'grad_norm': 18.568798065185547, 'learning_rate': 4.7168163495752977e-07, 'epoch': 0.36099810143748307, 'num_input_tokens_seen': 4469030912, 'completed': '57.80% (2_131 / 3_687)', 'remaining time': '13:51:12', 'throughput': '8226.47', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 03:19:25,652 >> {'loss': 0.5263, 'grad_norm': 21.5096492767334, 'learning_rate': 4.7128413906248885e-07, 'epoch': 0.36126932465419037, 'num_input_tokens_seen': 4471128064, 'completed': '57.82% (2_132 / 3_687)', 'remaining time': '13:50:40', 'throughput': '8258.02', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 03:19:55,545 >> {'loss': 0.3734, 'grad_norm': 14.280765533447266, 'learning_rate': 4.708867065149775e-07, 'epoch': 0.36154054787089773, 'num_input_tokens_seen': 4473225216, 'completed': '57.85% (2_133 / 3_687)', 'remaining time': '13:50:06', 'throughput': '8769.45', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 03:20:24,241 >> {'loss': 0.5905, 'grad_norm': 21.687335968017578, 'learning_rate': 4.704893376348342e-07, 'epoch': 0.3618117710876051, 'num_input_tokens_seen': 4475322368, 'completed': '57.88% (2_134 / 3_687)', 'remaining time': '13:49:30', 'throughput': '9135.08', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 03:20:51,885 >> {'loss': 0.6312, 'grad_norm': 18.599130630493164, 'learning_rate': 4.700920327418463e-07, 'epoch': 0.36208299430431246, 'num_input_tokens_seen': 4477419520, 'completed': '57.91% (2_135 / 3_687)', 'remaining time': '13:48:52', 'throughput': '9482.93', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 03:21:22,683 >> {'loss': 0.3828, 'grad_norm': 15.053811073303223, 'learning_rate': 4.696947921557498e-07, 'epoch': 0.3623542175210198, 'num_input_tokens_seen': 4479516672, 'completed': '57.93% (2_136 / 3_687)', 'remaining time': '13:48:19', 'throughput': '8511.76', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 03:21:52,835 >> {'loss': 0.443, 'grad_norm': 15.113408088684082, 'learning_rate': 4.6929761619622866e-07, 'epoch': 0.3626254407377271, 'num_input_tokens_seen': 4481613824, 'completed': '57.96% (2_137 / 3_687)', 'remaining time': '13:47:45', 'throughput': '8693.97', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 03:22:21,958 >> {'loss': 0.3773, 'grad_norm': 14.625730514526367, 'learning_rate': 4.68900505182915e-07, 'epoch': 0.3628966639544345, 'num_input_tokens_seen': 4483710976, 'completed': '57.99% (2_138 / 3_687)', 'remaining time': '13:47:09', 'throughput': '9001.40', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 03:22:53,336 >> {'loss': 0.2264, 'grad_norm': 13.605332374572754, 'learning_rate': 4.6850345943538896e-07, 'epoch': 0.36316788717114185, 'num_input_tokens_seen': 4485808128, 'completed': '58.01% (2_139 / 3_687)', 'remaining time': '13:46:36', 'throughput': '8354.37', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 03:23:25,162 >> {'loss': 0.5183, 'grad_norm': 18.40673065185547, 'learning_rate': 4.6810647927317735e-07, 'epoch': 0.3634391103878492, 'num_input_tokens_seen': 4487905280, 'completed': '58.04% (2_140 / 3_687)', 'remaining time': '13:46:04', 'throughput': '8236.90', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 03:23:55,546 >> {'loss': 0.4819, 'grad_norm': 17.36908721923828, 'learning_rate': 4.677095650157551e-07, 'epoch': 0.36371033360455657, 'num_input_tokens_seen': 4490002432, 'completed': '58.07% (2_141 / 3_687)', 'remaining time': '13:45:30', 'throughput': '8627.80', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 03:24:25,059 >> {'loss': 0.5503, 'grad_norm': 17.948476791381836, 'learning_rate': 4.6731271698254326e-07, 'epoch': 0.3639815568212639, 'num_input_tokens_seen': 4492099584, 'completed': '58.10% (2_142 / 3_687)', 'remaining time': '13:44:55', 'throughput': '8882.09', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 03:24:54,516 >> {'loss': 0.9079, 'grad_norm': 22.524932861328125, 'learning_rate': 4.669159354929105e-07, 'epoch': 0.36425278003797124, 'num_input_tokens_seen': 4494196736, 'completed': '58.12% (2_143 / 3_687)', 'remaining time': '13:44:20', 'throughput': '8899.23', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 03:25:24,284 >> {'loss': 0.2876, 'grad_norm': 11.158336639404297, 'learning_rate': 4.6651922086617134e-07, 'epoch': 0.3645240032546786, 'num_input_tokens_seen': 4496293888, 'completed': '58.15% (2_144 / 3_687)', 'remaining time': '13:43:46', 'throughput': '8806.29', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 03:25:56,082 >> {'loss': 0.4339, 'grad_norm': 16.561214447021484, 'learning_rate': 4.661225734215867e-07, 'epoch': 0.36479522647138596, 'num_input_tokens_seen': 4498391040, 'completed': '58.18% (2_145 / 3_687)', 'remaining time': '13:43:13', 'throughput': '8244.01', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 03:26:25,404 >> {'loss': 0.2669, 'grad_norm': 11.833779335021973, 'learning_rate': 4.657259934783628e-07, 'epoch': 0.3650664496880933, 'num_input_tokens_seen': 4500488192, 'completed': '58.20% (2_146 / 3_687)', 'remaining time': '13:42:38', 'throughput': '8940.15', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 03:26:53,304 >> {'loss': 0.5366, 'grad_norm': 15.685547828674316, 'learning_rate': 4.6532948135565264e-07, 'epoch': 0.3653376729048006, 'num_input_tokens_seen': 4502585344, 'completed': '58.23% (2_147 / 3_687)', 'remaining time': '13:42:01', 'throughput': '9395.75', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 03:27:25,489 >> {'loss': 0.4777, 'grad_norm': 17.699525833129883, 'learning_rate': 4.6493303737255397e-07, 'epoch': 0.365608896121508, 'num_input_tokens_seen': 4504682496, 'completed': '58.26% (2_148 / 3_687)', 'remaining time': '13:41:30', 'throughput': '8145.06', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 03:27:54,561 >> {'loss': 0.6268, 'grad_norm': 18.16535186767578, 'learning_rate': 4.645366618481098e-07, 'epoch': 0.36588011933821535, 'num_input_tokens_seen': 4506779648, 'completed': '58.29% (2_149 / 3_687)', 'remaining time': '13:40:54', 'throughput': '9017.05', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 03:28:23,520 >> {'loss': 0.729, 'grad_norm': 18.208988189697266, 'learning_rate': 4.641403551013081e-07, 'epoch': 0.3661513425549227, 'num_input_tokens_seen': 4508876800, 'completed': '58.31% (2_150 / 3_687)', 'remaining time': '13:40:19', 'throughput': '9052.05', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 03:28:56,487 >> {'loss': 0.3639, 'grad_norm': 14.158181190490723, 'learning_rate': 4.637441174510813e-07, 'epoch': 0.36642256577163007, 'num_input_tokens_seen': 4510973952, 'completed': '58.34% (2_151 / 3_687)', 'remaining time': '13:39:48', 'throughput': '7951.76', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 03:29:25,881 >> {'loss': 0.4579, 'grad_norm': 15.75287914276123, 'learning_rate': 4.633479492163066e-07, 'epoch': 0.3666937889883374, 'num_input_tokens_seen': 4513071104, 'completed': '58.37% (2_152 / 3_687)', 'remaining time': '13:39:13', 'throughput': '8918.27', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 03:29:56,410 >> {'loss': 0.413, 'grad_norm': 14.739073753356934, 'learning_rate': 4.6295185071580487e-07, 'epoch': 0.36696501220504474, 'num_input_tokens_seen': 4515168256, 'completed': '58.39% (2_153 / 3_687)', 'remaining time': '13:38:39', 'throughput': '8586.90', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 03:30:25,733 >> {'loss': 0.4131, 'grad_norm': 14.379277229309082, 'learning_rate': 4.6255582226834133e-07, 'epoch': 0.3672362354217521, 'num_input_tokens_seen': 4517265408, 'completed': '58.42% (2_154 / 3_687)', 'remaining time': '13:38:04', 'throughput': '8939.76', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 03:30:56,242 >> {'loss': 0.5776, 'grad_norm': 19.29395294189453, 'learning_rate': 4.6215986419262444e-07, 'epoch': 0.36750745863845946, 'num_input_tokens_seen': 4519362560, 'completed': '58.45% (2_155 / 3_687)', 'remaining time': '13:37:30', 'throughput': '8592.35', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 03:31:22,814 >> {'loss': 0.9065, 'grad_norm': 22.25945281982422, 'learning_rate': 4.617639768073066e-07, 'epoch': 0.3677786818551668, 'num_input_tokens_seen': 4521459712, 'completed': '58.48% (2_156 / 3_687)', 'remaining time': '13:36:52', 'throughput': '9865.36', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 03:31:53,797 >> {'loss': 0.302, 'grad_norm': 11.623411178588867, 'learning_rate': 4.613681604309824e-07, 'epoch': 0.36804990507187413, 'num_input_tokens_seen': 4523556864, 'completed': '58.50% (2_157 / 3_687)', 'remaining time': '13:36:19', 'throughput': '8460.91', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 03:32:24,896 >> {'loss': 0.5756, 'grad_norm': 19.57111358642578, 'learning_rate': 4.609724153821898e-07, 'epoch': 0.3683211282885815, 'num_input_tokens_seen': 4525654016, 'completed': '58.53% (2_158 / 3_687)', 'remaining time': '13:35:46', 'throughput': '8429.33', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 03:32:55,048 >> {'loss': 0.4861, 'grad_norm': 19.20432472229004, 'learning_rate': 4.6057674197940974e-07, 'epoch': 0.36859235150528885, 'num_input_tokens_seen': 4527751168, 'completed': '58.56% (2_159 / 3_687)', 'remaining time': '13:35:12', 'throughput': '8694.02', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 03:33:28,406 >> {'loss': 0.3036, 'grad_norm': 12.728202819824219, 'learning_rate': 4.6018114054106494e-07, 'epoch': 0.3688635747219962, 'num_input_tokens_seen': 4529848320, 'completed': '58.58% (2_160 / 3_687)', 'remaining time': '13:34:41', 'throughput': '7858.58', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 03:33:58,361 >> {'loss': 0.3981, 'grad_norm': 15.91971492767334, 'learning_rate': 4.597856113855203e-07, 'epoch': 0.3691347979387036, 'num_input_tokens_seen': 4531945472, 'completed': '58.61% (2_161 / 3_687)', 'remaining time': '13:34:07', 'throughput': '8751.31', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 03:34:25,755 >> {'loss': 0.6073, 'grad_norm': 20.145366668701172, 'learning_rate': 4.593901548310828e-07, 'epoch': 0.3694060211554109, 'num_input_tokens_seen': 4534042624, 'completed': '58.64% (2_162 / 3_687)', 'remaining time': '13:33:30', 'throughput': '9569.33', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 03:34:53,845 >> {'loss': 0.4431, 'grad_norm': 18.48992156982422, 'learning_rate': 4.5899477119600073e-07, 'epoch': 0.36967724437211824, 'num_input_tokens_seen': 4536139776, 'completed': '58.67% (2_163 / 3_687)', 'remaining time': '13:32:53', 'throughput': '9332.36', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 03:35:27,993 >> {'loss': 0.4718, 'grad_norm': 20.453821182250977, 'learning_rate': 4.585994607984637e-07, 'epoch': 0.3699484675888256, 'num_input_tokens_seen': 4538236928, 'completed': '58.69% (2_164 / 3_687)', 'remaining time': '13:32:24', 'throughput': '7676.63', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 03:35:59,442 >> {'loss': 0.555, 'grad_norm': 19.20899772644043, 'learning_rate': 4.582042239566026e-07, 'epoch': 0.37021969080553296, 'num_input_tokens_seen': 4540334080, 'completed': '58.72% (2_165 / 3_687)', 'remaining time': '13:31:51', 'throughput': '8335.55', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 03:36:35,410 >> {'loss': 0.3494, 'grad_norm': 14.162814140319824, 'learning_rate': 4.578090609884889e-07, 'epoch': 0.3704909140222403, 'num_input_tokens_seen': 4542431232, 'completed': '58.75% (2_166 / 3_687)', 'remaining time': '13:31:23', 'throughput': '7288.19', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 03:37:05,591 >> {'loss': 0.3046, 'grad_norm': 16.07235336303711, 'learning_rate': 4.57413972212135e-07, 'epoch': 0.37076213723894763, 'num_input_tokens_seen': 4544528384, 'completed': '58.77% (2_167 / 3_687)', 'remaining time': '13:30:49', 'throughput': '8685.81', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 03:37:34,698 >> {'loss': 0.403, 'grad_norm': 18.295516967773438, 'learning_rate': 4.5701895794549293e-07, 'epoch': 0.371033360455655, 'num_input_tokens_seen': 4546625536, 'completed': '58.80% (2_168 / 3_687)', 'remaining time': '13:30:14', 'throughput': '9006.33', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 03:38:06,775 >> {'loss': 0.3592, 'grad_norm': 19.14270782470703, 'learning_rate': 4.566240185064554e-07, 'epoch': 0.37130458367236235, 'num_input_tokens_seen': 4548722688, 'completed': '58.83% (2_169 / 3_687)', 'remaining time': '13:29:42', 'throughput': '8172.25', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 03:38:39,232 >> {'loss': 0.5408, 'grad_norm': 17.336620330810547, 'learning_rate': 4.5622915421285446e-07, 'epoch': 0.3715758068890697, 'num_input_tokens_seen': 4550819840, 'completed': '58.86% (2_170 / 3_687)', 'remaining time': '13:29:11', 'throughput': '8076.64', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 03:39:09,038 >> {'loss': 0.0982, 'grad_norm': 7.247825622558594, 'learning_rate': 4.558343653824619e-07, 'epoch': 0.3718470301057771, 'num_input_tokens_seen': 4552916992, 'completed': '58.88% (2_171 / 3_687)', 'remaining time': '13:28:36', 'throughput': '8795.12', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 03:39:39,899 >> {'loss': 0.3333, 'grad_norm': 14.099793434143066, 'learning_rate': 4.55439652332989e-07, 'epoch': 0.3721182533224844, 'num_input_tokens_seen': 4555014144, 'completed': '58.91% (2_172 / 3_687)', 'remaining time': '13:28:03', 'throughput': '8494.28', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 03:40:11,025 >> {'loss': 0.4548, 'grad_norm': 17.59613800048828, 'learning_rate': 4.550450153820857e-07, 'epoch': 0.37238947653919174, 'num_input_tokens_seen': 4557111296, 'completed': '58.94% (2_173 / 3_687)', 'remaining time': '13:27:30', 'throughput': '8422.02', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 03:40:43,235 >> {'loss': 0.3559, 'grad_norm': 12.305207252502441, 'learning_rate': 4.5465045484734044e-07, 'epoch': 0.3726606997558991, 'num_input_tokens_seen': 4559208448, 'completed': '58.96% (2_174 / 3_687)', 'remaining time': '13:26:58', 'throughput': '8138.55', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 03:41:12,505 >> {'loss': 0.3236, 'grad_norm': 13.379070281982422, 'learning_rate': 4.5425597104628086e-07, 'epoch': 0.37293192297260647, 'num_input_tokens_seen': 4561305600, 'completed': '58.99% (2_175 / 3_687)', 'remaining time': '13:26:23', 'throughput': '8956.22', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 03:41:41,082 >> {'loss': 0.4341, 'grad_norm': 15.527801513671875, 'learning_rate': 4.5386156429637256e-07, 'epoch': 0.37320314618931383, 'num_input_tokens_seen': 4563402752, 'completed': '59.02% (2_176 / 3_687)', 'remaining time': '13:25:48', 'throughput': '9173.26', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 03:42:14,281 >> {'loss': 0.3883, 'grad_norm': 22.03895378112793, 'learning_rate': 4.534672349150189e-07, 'epoch': 0.37347436940602113, 'num_input_tokens_seen': 4565499904, 'completed': '59.05% (2_177 / 3_687)', 'remaining time': '13:25:17', 'throughput': '7895.98', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 03:42:44,417 >> {'loss': 0.311, 'grad_norm': 15.303971290588379, 'learning_rate': 4.530729832195612e-07, 'epoch': 0.3737455926227285, 'num_input_tokens_seen': 4567597056, 'completed': '59.07% (2_178 / 3_687)', 'remaining time': '13:24:43', 'throughput': '8698.66', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 03:43:16,664 >> {'loss': 0.3807, 'grad_norm': 15.900908470153809, 'learning_rate': 4.5267880952727845e-07, 'epoch': 0.37401681583943586, 'num_input_tokens_seen': 4569694208, 'completed': '59.10% (2_179 / 3_687)', 'remaining time': '13:24:11', 'throughput': '8129.40', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 03:43:44,421 >> {'loss': 0.9865, 'grad_norm': 26.075777053833008, 'learning_rate': 4.5228471415538637e-07, 'epoch': 0.3742880390561432, 'num_input_tokens_seen': 4571791360, 'completed': '59.13% (2_180 / 3_687)', 'remaining time': '13:23:35', 'throughput': '9444.22', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 03:44:15,566 >> {'loss': 0.3947, 'grad_norm': 18.148820877075195, 'learning_rate': 4.518906974210379e-07, 'epoch': 0.3745592622728506, 'num_input_tokens_seen': 4573888512, 'completed': '59.15% (2_181 / 3_687)', 'remaining time': '13:23:02', 'throughput': '8416.86', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 03:44:45,447 >> {'loss': 0.4087, 'grad_norm': 32.52322006225586, 'learning_rate': 4.514967596413228e-07, 'epoch': 0.3748304854895579, 'num_input_tokens_seen': 4575985664, 'completed': '59.18% (2_182 / 3_687)', 'remaining time': '13:22:27', 'throughput': '8772.87', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 03:45:15,887 >> {'loss': 0.3748, 'grad_norm': 14.72100830078125, 'learning_rate': 4.511029011332672e-07, 'epoch': 0.37510170870626525, 'num_input_tokens_seen': 4578082816, 'completed': '59.21% (2_183 / 3_687)', 'remaining time': '13:21:54', 'throughput': '8612.02', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 03:45:47,047 >> {'loss': 0.5725, 'grad_norm': 18.871889114379883, 'learning_rate': 4.507091222138335e-07, 'epoch': 0.3753729319229726, 'num_input_tokens_seen': 4580179968, 'completed': '59.24% (2_184 / 3_687)', 'remaining time': '13:21:21', 'throughput': '8412.64', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 03:46:17,793 >> {'loss': 0.2177, 'grad_norm': 15.662956237792969, 'learning_rate': 4.503154231999202e-07, 'epoch': 0.37564415513967997, 'num_input_tokens_seen': 4582277120, 'completed': '59.26% (2_185 / 3_687)', 'remaining time': '13:20:47', 'throughput': '8526.22', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 03:46:47,673 >> {'loss': 0.799, 'grad_norm': 23.3785343170166, 'learning_rate': 4.499218044083608e-07, 'epoch': 0.37591537835638733, 'num_input_tokens_seen': 4584374272, 'completed': '59.29% (2_186 / 3_687)', 'remaining time': '13:20:13', 'throughput': '8773.24', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 03:47:18,729 >> {'loss': 0.4064, 'grad_norm': 15.690712928771973, 'learning_rate': 4.495282661559248e-07, 'epoch': 0.37618660157309464, 'num_input_tokens_seen': 4586471424, 'completed': '59.32% (2_187 / 3_687)', 'remaining time': '13:19:40', 'throughput': '8441.11', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 03:47:47,776 >> {'loss': 0.5982, 'grad_norm': 17.365020751953125, 'learning_rate': 4.491348087593171e-07, 'epoch': 0.376457824789802, 'num_input_tokens_seen': 4588568576, 'completed': '59.34% (2_188 / 3_687)', 'remaining time': '13:19:05', 'throughput': '9024.81', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 03:48:17,890 >> {'loss': 0.2989, 'grad_norm': 12.830045700073242, 'learning_rate': 4.4874143253517706e-07, 'epoch': 0.37672904800650936, 'num_input_tokens_seen': 4590665728, 'completed': '59.37% (2_189 / 3_687)', 'remaining time': '13:18:31', 'throughput': '8704.82', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 03:48:48,676 >> {'loss': 0.7782, 'grad_norm': 21.898000717163086, 'learning_rate': 4.483481378000791e-07, 'epoch': 0.3770002712232167, 'num_input_tokens_seen': 4592762880, 'completed': '59.40% (2_190 / 3_687)', 'remaining time': '13:17:58', 'throughput': '8514.99', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 03:49:19,928 >> {'loss': 0.5265, 'grad_norm': 17.4901065826416, 'learning_rate': 4.4795492487053155e-07, 'epoch': 0.3772714944399241, 'num_input_tokens_seen': 4594860032, 'completed': '59.43% (2_191 / 3_687)', 'remaining time': '13:17:25', 'throughput': '8388.13', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 03:49:51,613 >> {'loss': 0.3187, 'grad_norm': 11.635929107666016, 'learning_rate': 4.4756179406297744e-07, 'epoch': 0.3775427176566314, 'num_input_tokens_seen': 4596957184, 'completed': '59.45% (2_192 / 3_687)', 'remaining time': '13:16:53', 'throughput': '8273.47', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 03:50:20,547 >> {'loss': 0.3864, 'grad_norm': 16.157140731811523, 'learning_rate': 4.4716874569379336e-07, 'epoch': 0.37781394087333875, 'num_input_tokens_seen': 4599054336, 'completed': '59.48% (2_193 / 3_687)', 'remaining time': '13:16:17', 'throughput': '9060.17', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 03:50:47,885 >> {'loss': 0.3689, 'grad_norm': 15.90643310546875, 'learning_rate': 4.467757800792897e-07, 'epoch': 0.3780851640900461, 'num_input_tokens_seen': 4601151488, 'completed': '59.51% (2_194 / 3_687)', 'remaining time': '13:15:40', 'throughput': '9588.96', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 03:51:17,153 >> {'loss': 0.6203, 'grad_norm': 20.70597267150879, 'learning_rate': 4.4638289753571025e-07, 'epoch': 0.37835638730675347, 'num_input_tokens_seen': 4603248640, 'completed': '59.53% (2_195 / 3_687)', 'remaining time': '13:15:06', 'throughput': '8956.74', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 03:51:47,613 >> {'loss': 0.3584, 'grad_norm': 13.205007553100586, 'learning_rate': 4.459900983792321e-07, 'epoch': 0.37862761052346083, 'num_input_tokens_seen': 4605345792, 'completed': '59.56% (2_196 / 3_687)', 'remaining time': '13:14:32', 'throughput': '8606.19', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 03:52:18,480 >> {'loss': 0.2914, 'grad_norm': 15.028583526611328, 'learning_rate': 4.455973829259648e-07, 'epoch': 0.37889883374016814, 'num_input_tokens_seen': 4607442944, 'completed': '59.59% (2_197 / 3_687)', 'remaining time': '13:13:59', 'throughput': '8492.49', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 03:52:50,750 >> {'loss': 0.4295, 'grad_norm': 14.172557830810547, 'learning_rate': 4.4520475149195093e-07, 'epoch': 0.3791700569568755, 'num_input_tokens_seen': 4609540096, 'completed': '59.61% (2_198 / 3_687)', 'remaining time': '13:13:27', 'throughput': '8123.48', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 03:53:20,198 >> {'loss': 0.2735, 'grad_norm': 11.871790885925293, 'learning_rate': 4.4481220439316514e-07, 'epoch': 0.37944128017358286, 'num_input_tokens_seen': 4611637248, 'completed': '59.64% (2_199 / 3_687)', 'remaining time': '13:12:52', 'throughput': '8901.95', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 03:53:50,365 >> {'loss': 0.6647, 'grad_norm': 17.767791748046875, 'learning_rate': 4.444197419455147e-07, 'epoch': 0.3797125033902902, 'num_input_tokens_seen': 4613734400, 'completed': '59.67% (2_200 / 3_687)', 'remaining time': '13:12:19', 'throughput': '8689.80', 'gpu_mem_free': '5581MB'} +/scratch3/workspace/ctpham_umass_edu-ft/envs/prolong-final/lib/python3.10/site-packages/torch/distributed/fsdp/fully_sharded_data_parallel.py:689: FutureWarning: FSDP.state_dict_type() and FSDP.set_state_dict_type() are being deprecated. Please use APIs, get_state_dict() and set_state_dict(), which can support different parallelisms, FSDP1, FSDP2, DDP. API doc: https://pytorch.org/docs/stable/distributed.checkpoint.html#torch.distributed.checkpoint.state_dict.get_state_dict .Tutorial: https://pytorch.org/tutorials/recipes/distributed_checkpoint_recipe.html . + warnings.warn( +[INFO|trainer.py:3503] 2024-12-26 03:54:13,820 >> Saving model checkpoint to /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/checkpoint-2200 +[INFO|configuration_utils.py:472] 2024-12-26 03:54:13,823 >> Configuration saved in /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/checkpoint-2200/config.json +[INFO|configuration_utils.py:807] 2024-12-26 03:54:13,825 >> Configuration saved in /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/checkpoint-2200/generation_config.json +[INFO|modeling_utils.py:2807] 2024-12-26 03:55:09,960 >> The model is bigger than the maximum size per checkpoint (5GB) and is going to be split in 7 checkpoint shards. You can find where each parameters has been saved in the index located at /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/checkpoint-2200/model.safetensors.index.json. +[INFO|tokenization_utils_base.py:2684] 2024-12-26 03:55:09,964 >> tokenizer config file saved in /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/checkpoint-2200/tokenizer_config.json +[INFO|tokenization_utils_base.py:2693] 2024-12-26 03:55:09,965 >> Special tokens file saved in /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/checkpoint-2200/special_tokens_map.json +/scratch3/workspace/ctpham_umass_edu-ft/envs/prolong-final/lib/python3.10/site-packages/torch/distributed/fsdp/fully_sharded_data_parallel.py:689: FutureWarning: FSDP.state_dict_type() and FSDP.set_state_dict_type() are being deprecated. Please use APIs, get_state_dict() and set_state_dict(), which can support different parallelisms, FSDP1, FSDP2, DDP. API doc: https://pytorch.org/docs/stable/distributed.checkpoint.html#torch.distributed.checkpoint.state_dict.get_state_dict .Tutorial: https://pytorch.org/tutorials/recipes/distributed_checkpoint_recipe.html . + warnings.warn( +[WARNING|trainer.py:868] 2024-12-26 03:59:00,091 >> Save streaming dataset state: {'epoch': 0, 'sample_in_epoch': 4400, 'num_canonical_nodes': 1, 'shuffle_seed': 42, 'initial_physical_nodes': 1} +/scratch3/workspace/ctpham_umass_edu-ft/envs/prolong-final/lib/python3.10/site-packages/torch/utils/checkpoint.py:1399: FutureWarning: `torch.cpu.amp.autocast(args...)` is deprecated. Please use `torch.amp.autocast('cpu', args...)` instead. + with device_autocast_ctx, torch.cpu.amp.autocast(**cpu_autocast_kwargs), recompute_context: # type: ignore[attr-defined] +[INFO|trainer.py:175] 2024-12-26 03:59:32,063 >> {'loss': 0.4934, 'grad_norm': 15.653825759887695, 'learning_rate': 4.4402736446483824e-07, 'epoch': 0.3799837266069976, 'num_input_tokens_seen': 4615831552, 'completed': '59.70% (2_201 / 3_687)', 'remaining time': '13:17:15', 'throughput': '767.18', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 04:00:05,569 >> {'loss': 0.4065, 'grad_norm': 16.383865356445312, 'learning_rate': 4.436350722669065e-07, 'epoch': 0.3802549498237049, 'num_input_tokens_seen': 4617928704, 'completed': '59.72% (2_202 / 3_687)', 'remaining time': '13:16:44', 'throughput': '7823.91', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 04:00:34,089 >> {'loss': 0.5536, 'grad_norm': 17.470500946044922, 'learning_rate': 4.43242865667421e-07, 'epoch': 0.38052617304041225, 'num_input_tokens_seen': 4620025856, 'completed': '59.75% (2_203 / 3_687)', 'remaining time': '13:16:08', 'throughput': '9191.38', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 04:01:05,211 >> {'loss': 0.4773, 'grad_norm': 17.186763763427734, 'learning_rate': 4.428507449820147e-07, 'epoch': 0.3807973962571196, 'num_input_tokens_seen': 4622123008, 'completed': '59.78% (2_204 / 3_687)', 'remaining time': '13:15:35', 'throughput': '8423.09', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 04:01:33,843 >> {'loss': 0.6064, 'grad_norm': 18.592941284179688, 'learning_rate': 4.424587105262516e-07, 'epoch': 0.381068619473827, 'num_input_tokens_seen': 4624220160, 'completed': '59.80% (2_205 / 3_687)', 'remaining time': '13:14:59', 'throughput': '9155.62', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 04:02:00,907 >> {'loss': 0.6243, 'grad_norm': 18.531604766845703, 'learning_rate': 4.4206676261562603e-07, 'epoch': 0.38133984269053434, 'num_input_tokens_seen': 4626317312, 'completed': '59.83% (2_206 / 3_687)', 'remaining time': '13:14:21', 'throughput': '9686.36', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 04:02:31,693 >> {'loss': 0.3981, 'grad_norm': 18.356502532958984, 'learning_rate': 4.4167490156556274e-07, 'epoch': 0.38161106590724164, 'num_input_tokens_seen': 4628414464, 'completed': '59.86% (2_207 / 3_687)', 'remaining time': '13:13:48', 'throughput': '8515.02', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 04:03:01,766 >> {'loss': 0.4111, 'grad_norm': 16.479177474975586, 'learning_rate': 4.4128312769141664e-07, 'epoch': 0.381882289123949, 'num_input_tokens_seen': 4630511616, 'completed': '59.89% (2_208 / 3_687)', 'remaining time': '13:13:13', 'throughput': '8716.87', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 04:03:32,290 >> {'loss': 0.3192, 'grad_norm': 13.391749382019043, 'learning_rate': 4.408914413084722e-07, 'epoch': 0.38215351234065636, 'num_input_tokens_seen': 4632608768, 'completed': '59.91% (2_209 / 3_687)', 'remaining time': '13:12:39', 'throughput': '8588.21', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 04:04:03,567 >> {'loss': 0.7114, 'grad_norm': 24.691661834716797, 'learning_rate': 4.4049984273194406e-07, 'epoch': 0.3824247355573637, 'num_input_tokens_seen': 4634705920, 'completed': '59.94% (2_210 / 3_687)', 'remaining time': '13:12:06', 'throughput': '8381.32', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 04:04:33,226 >> {'loss': 0.7462, 'grad_norm': 27.44068717956543, 'learning_rate': 4.401083322769756e-07, 'epoch': 0.3826959587740711, 'num_input_tokens_seen': 4636803072, 'completed': '59.97% (2_211 / 3_687)', 'remaining time': '13:11:32', 'throughput': '8838.55', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 04:05:04,595 >> {'loss': 0.4238, 'grad_norm': 14.547937393188477, 'learning_rate': 4.3971691025863964e-07, 'epoch': 0.3829671819907784, 'num_input_tokens_seen': 4638900224, 'completed': '59.99% (2_212 / 3_687)', 'remaining time': '13:10:59', 'throughput': '8356.81', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 04:05:34,842 >> {'loss': 0.5338, 'grad_norm': 16.86309242248535, 'learning_rate': 4.3932557699193794e-07, 'epoch': 0.38323840520748575, 'num_input_tokens_seen': 4640997376, 'completed': '60.02% (2_213 / 3_687)', 'remaining time': '13:10:24', 'throughput': '8666.64', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 04:06:05,326 >> {'loss': 0.3702, 'grad_norm': 13.788909912109375, 'learning_rate': 4.389343327918004e-07, 'epoch': 0.3835096284241931, 'num_input_tokens_seen': 4643094528, 'completed': '60.05% (2_214 / 3_687)', 'remaining time': '13:09:50', 'throughput': '8599.56', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 04:06:35,588 >> {'loss': 0.4389, 'grad_norm': 18.939809799194336, 'learning_rate': 4.3854317797308515e-07, 'epoch': 0.3837808516409005, 'num_input_tokens_seen': 4645191680, 'completed': '60.08% (2_215 / 3_687)', 'remaining time': '13:09:16', 'throughput': '8662.37', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 04:07:07,720 >> {'loss': 0.3214, 'grad_norm': 13.284666061401367, 'learning_rate': 4.381521128505791e-07, 'epoch': 0.3840520748576078, 'num_input_tokens_seen': 4647288832, 'completed': '60.10% (2_216 / 3_687)', 'remaining time': '13:08:44', 'throughput': '8158.42', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 04:07:39,203 >> {'loss': 0.2278, 'grad_norm': 10.879389762878418, 'learning_rate': 4.377611377389966e-07, 'epoch': 0.38432329807431515, 'num_input_tokens_seen': 4649385984, 'completed': '60.13% (2_217 / 3_687)', 'remaining time': '13:08:11', 'throughput': '8326.42', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 04:08:11,830 >> {'loss': 0.3289, 'grad_norm': 12.832559585571289, 'learning_rate': 4.3737025295297937e-07, 'epoch': 0.3845945212910225, 'num_input_tokens_seen': 4651483136, 'completed': '60.16% (2_218 / 3_687)', 'remaining time': '13:07:39', 'throughput': '8034.70', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 04:08:42,943 >> {'loss': 0.5121, 'grad_norm': 15.852336883544922, 'learning_rate': 4.3697945880709686e-07, 'epoch': 0.38486574450772987, 'num_input_tokens_seen': 4653580288, 'completed': '60.18% (2_219 / 3_687)', 'remaining time': '13:07:06', 'throughput': '8425.48', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 04:09:14,306 >> {'loss': 0.5452, 'grad_norm': 19.836444854736328, 'learning_rate': 4.3658875561584494e-07, 'epoch': 0.38513696772443723, 'num_input_tokens_seen': 4655677440, 'completed': '60.21% (2_220 / 3_687)', 'remaining time': '13:06:33', 'throughput': '8358.34', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 04:09:45,939 >> {'loss': 0.1762, 'grad_norm': 18.720237731933594, 'learning_rate': 4.36198143693647e-07, 'epoch': 0.38540819094114454, 'num_input_tokens_seen': 4657774592, 'completed': '60.24% (2_221 / 3_687)', 'remaining time': '13:06:00', 'throughput': '8286.97', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 04:10:16,051 >> {'loss': 0.2582, 'grad_norm': 12.053540229797363, 'learning_rate': 4.3580762335485253e-07, 'epoch': 0.3856794141578519, 'num_input_tokens_seen': 4659871744, 'completed': '60.27% (2_222 / 3_687)', 'remaining time': '13:05:26', 'throughput': '8705.85', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 04:10:47,620 >> {'loss': 0.7018, 'grad_norm': 21.45743751525879, 'learning_rate': 4.3541719491373743e-07, 'epoch': 0.38595063737455926, 'num_input_tokens_seen': 4661968896, 'completed': '60.29% (2_223 / 3_687)', 'remaining time': '13:04:53', 'throughput': '8303.71', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 04:11:19,425 >> {'loss': 0.686, 'grad_norm': 18.61931610107422, 'learning_rate': 4.350268586845035e-07, 'epoch': 0.3862218605912666, 'num_input_tokens_seen': 4664066048, 'completed': '60.32% (2_224 / 3_687)', 'remaining time': '13:04:21', 'throughput': '8242.14', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 04:11:50,991 >> {'loss': 0.4922, 'grad_norm': 15.874135971069336, 'learning_rate': 4.34636614981279e-07, 'epoch': 0.386493083807974, 'num_input_tokens_seen': 4666163200, 'completed': '60.35% (2_225 / 3_687)', 'remaining time': '13:03:48', 'throughput': '8304.72', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 04:12:20,366 >> {'loss': 0.4819, 'grad_norm': 17.982471466064453, 'learning_rate': 4.342464641181166e-07, 'epoch': 0.3867643070246813, 'num_input_tokens_seen': 4668260352, 'completed': '60.37% (2_226 / 3_687)', 'remaining time': '13:03:13', 'throughput': '8924.12', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 04:12:50,318 >> {'loss': 0.3803, 'grad_norm': 13.340551376342773, 'learning_rate': 4.3385640640899524e-07, 'epoch': 0.38703553024138865, 'num_input_tokens_seen': 4670357504, 'completed': '60.40% (2_227 / 3_687)', 'remaining time': '13:02:39', 'throughput': '8752.12', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 04:13:20,902 >> {'loss': 0.4039, 'grad_norm': 15.015870094299316, 'learning_rate': 4.3346644216781823e-07, 'epoch': 0.387306753458096, 'num_input_tokens_seen': 4672454656, 'completed': '60.43% (2_228 / 3_687)', 'remaining time': '13:02:05', 'throughput': '8571.41', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 04:13:51,402 >> {'loss': 0.2899, 'grad_norm': 13.041723251342773, 'learning_rate': 4.3307657170841417e-07, 'epoch': 0.38757797667480337, 'num_input_tokens_seen': 4674551808, 'completed': '60.46% (2_229 / 3_687)', 'remaining time': '13:01:31', 'throughput': '8594.84', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 04:14:22,499 >> {'loss': 0.7029, 'grad_norm': 21.264972686767578, 'learning_rate': 4.326867953445362e-07, 'epoch': 0.38784919989151073, 'num_input_tokens_seen': 4676648960, 'completed': '60.48% (2_230 / 3_687)', 'remaining time': '13:00:58', 'throughput': '8429.74', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 04:14:55,637 >> {'loss': 0.4447, 'grad_norm': 17.26335906982422, 'learning_rate': 4.322971133898611e-07, 'epoch': 0.38812042310821804, 'num_input_tokens_seen': 4678746112, 'completed': '60.51% (2_231 / 3_687)', 'remaining time': '13:00:27', 'throughput': '7910.84', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 04:15:24,050 >> {'loss': 0.3625, 'grad_norm': 13.549259185791016, 'learning_rate': 4.319075261579901e-07, 'epoch': 0.3883916463249254, 'num_input_tokens_seen': 4680843264, 'completed': '60.54% (2_232 / 3_687)', 'remaining time': '12:59:51', 'throughput': '9225.94', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 04:15:52,690 >> {'loss': 0.5931, 'grad_norm': 17.16545295715332, 'learning_rate': 4.3151803396244833e-07, 'epoch': 0.38866286954163276, 'num_input_tokens_seen': 4682940416, 'completed': '60.56% (2_233 / 3_687)', 'remaining time': '12:59:15', 'throughput': '9153.03', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 04:16:23,526 >> {'loss': 0.3044, 'grad_norm': 20.503673553466797, 'learning_rate': 4.3112863711668435e-07, 'epoch': 0.3889340927583401, 'num_input_tokens_seen': 4685037568, 'completed': '60.59% (2_234 / 3_687)', 'remaining time': '12:58:41', 'throughput': '8501.50', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 04:16:54,225 >> {'loss': 0.9052, 'grad_norm': 24.06279182434082, 'learning_rate': 4.3073933593406974e-07, 'epoch': 0.3892053159750475, 'num_input_tokens_seen': 4687134720, 'completed': '60.62% (2_235 / 3_687)', 'remaining time': '12:58:08', 'throughput': '8539.17', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 04:17:23,292 >> {'loss': 0.3166, 'grad_norm': 13.339831352233887, 'learning_rate': 4.303501307278994e-07, 'epoch': 0.3894765391917548, 'num_input_tokens_seen': 4689231872, 'completed': '60.65% (2_236 / 3_687)', 'remaining time': '12:57:32', 'throughput': '9018.42', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 04:17:52,786 >> {'loss': 0.4647, 'grad_norm': 14.58847713470459, 'learning_rate': 4.299610218113908e-07, 'epoch': 0.38974776240846215, 'num_input_tokens_seen': 4691329024, 'completed': '60.67% (2_237 / 3_687)', 'remaining time': '12:56:58', 'throughput': '8888.18', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 04:18:26,221 >> {'loss': 0.3683, 'grad_norm': 16.615280151367188, 'learning_rate': 4.2957200949768414e-07, 'epoch': 0.3900189856251695, 'num_input_tokens_seen': 4693426176, 'completed': '60.70% (2_238 / 3_687)', 'remaining time': '12:56:27', 'throughput': '7840.31', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 04:18:57,851 >> {'loss': 0.6683, 'grad_norm': 25.591224670410156, 'learning_rate': 4.2918309409984145e-07, 'epoch': 0.3902902088418769, 'num_input_tokens_seen': 4695523328, 'completed': '60.73% (2_239 / 3_687)', 'remaining time': '12:55:54', 'throughput': '8287.86', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 04:19:29,708 >> {'loss': 0.1616, 'grad_norm': 9.293283462524414, 'learning_rate': 4.2879427593084714e-07, 'epoch': 0.39056143205858423, 'num_input_tokens_seen': 4697620480, 'completed': '60.75% (2_240 / 3_687)', 'remaining time': '12:55:22', 'throughput': '8228.68', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 04:20:02,239 >> {'loss': 0.3878, 'grad_norm': 16.830848693847656, 'learning_rate': 4.2840555530360756e-07, 'epoch': 0.39083265527529154, 'num_input_tokens_seen': 4699717632, 'completed': '60.78% (2_241 / 3_687)', 'remaining time': '12:54:50', 'throughput': '8058.33', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 04:20:31,788 >> {'loss': 0.3504, 'grad_norm': 13.508627891540527, 'learning_rate': 4.280169325309502e-07, 'epoch': 0.3911038784919989, 'num_input_tokens_seen': 4701814784, 'completed': '60.81% (2_242 / 3_687)', 'remaining time': '12:54:15', 'throughput': '8871.61', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 04:21:03,206 >> {'loss': 0.2894, 'grad_norm': 12.19189739227295, 'learning_rate': 4.276284079256235e-07, 'epoch': 0.39137510170870626, 'num_input_tokens_seen': 4703911936, 'completed': '60.84% (2_243 / 3_687)', 'remaining time': '12:53:42', 'throughput': '8343.76', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 04:21:33,547 >> {'loss': 0.5448, 'grad_norm': 24.510799407958984, 'learning_rate': 4.272399818002974e-07, 'epoch': 0.3916463249254136, 'num_input_tokens_seen': 4706009088, 'completed': '60.86% (2_244 / 3_687)', 'remaining time': '12:53:08', 'throughput': '8639.90', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 04:22:03,652 >> {'loss': 0.2909, 'grad_norm': 17.75606918334961, 'learning_rate': 4.268516544675628e-07, 'epoch': 0.391917548142121, 'num_input_tokens_seen': 4708106240, 'completed': '60.89% (2_245 / 3_687)', 'remaining time': '12:52:34', 'throughput': '8707.63', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 04:22:37,489 >> {'loss': 0.3659, 'grad_norm': 16.183746337890625, 'learning_rate': 4.2646342623993035e-07, 'epoch': 0.3921887713588283, 'num_input_tokens_seen': 4710203392, 'completed': '60.92% (2_246 / 3_687)', 'remaining time': '12:52:04', 'throughput': '7747.20', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 04:23:06,349 >> {'loss': 0.3025, 'grad_norm': 12.006651878356934, 'learning_rate': 4.2607529742983174e-07, 'epoch': 0.39245999457553565, 'num_input_tokens_seen': 4712300544, 'completed': '60.94% (2_247 / 3_687)', 'remaining time': '12:51:28', 'throughput': '9083.38', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 04:23:34,924 >> {'loss': 0.6958, 'grad_norm': 22.091772079467773, 'learning_rate': 4.2568726834961797e-07, 'epoch': 0.392731217792243, 'num_input_tokens_seen': 4714397696, 'completed': '60.97% (2_248 / 3_687)', 'remaining time': '12:50:53', 'throughput': '9173.82', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 04:24:05,734 >> {'loss': 0.2623, 'grad_norm': 13.590865135192871, 'learning_rate': 4.252993393115601e-07, 'epoch': 0.3930024410089504, 'num_input_tokens_seen': 4716494848, 'completed': '61.00% (2_249 / 3_687)', 'remaining time': '12:50:19', 'throughput': '8508.58', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 04:24:35,096 >> {'loss': 0.4497, 'grad_norm': 18.128318786621094, 'learning_rate': 4.249115106278489e-07, 'epoch': 0.39327366422565774, 'num_input_tokens_seen': 4718592000, 'completed': '61.03% (2_250 / 3_687)', 'remaining time': '12:49:44', 'throughput': '8928.02', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 04:25:04,693 >> {'loss': 0.5144, 'grad_norm': 17.97321319580078, 'learning_rate': 4.2452378261059397e-07, 'epoch': 0.39354488744236504, 'num_input_tokens_seen': 4720689152, 'completed': '61.05% (2_251 / 3_687)', 'remaining time': '12:49:09', 'throughput': '8857.06', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 04:25:33,608 >> {'loss': 0.4629, 'grad_norm': 17.074485778808594, 'learning_rate': 4.24136155571824e-07, 'epoch': 0.3938161106590724, 'num_input_tokens_seen': 4722786304, 'completed': '61.08% (2_252 / 3_687)', 'remaining time': '12:48:34', 'throughput': '9065.77', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 04:26:05,313 >> {'loss': 0.4546, 'grad_norm': 16.58645248413086, 'learning_rate': 4.2374862982348657e-07, 'epoch': 0.39408733387577977, 'num_input_tokens_seen': 4724883456, 'completed': '61.11% (2_253 / 3_687)', 'remaining time': '12:48:02', 'throughput': '8268.46', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 04:26:35,855 >> {'loss': 0.8026, 'grad_norm': 20.560569763183594, 'learning_rate': 4.233612056774477e-07, 'epoch': 0.3943585570924871, 'num_input_tokens_seen': 4726980608, 'completed': '61.13% (2_254 / 3_687)', 'remaining time': '12:47:28', 'throughput': '8582.95', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 04:27:06,351 >> {'loss': 0.2919, 'grad_norm': 13.33758544921875, 'learning_rate': 4.2297388344549146e-07, 'epoch': 0.3946297803091945, 'num_input_tokens_seen': 4729077760, 'completed': '61.16% (2_255 / 3_687)', 'remaining time': '12:46:54', 'throughput': '8595.95', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 04:27:36,729 >> {'loss': 0.4084, 'grad_norm': 17.15109634399414, 'learning_rate': 4.2258666343932004e-07, 'epoch': 0.3949010035259018, 'num_input_tokens_seen': 4731174912, 'completed': '61.19% (2_256 / 3_687)', 'remaining time': '12:46:20', 'throughput': '8629.37', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 04:28:05,930 >> {'loss': 0.365, 'grad_norm': 16.682199478149414, 'learning_rate': 4.2219954597055354e-07, 'epoch': 0.39517222674260916, 'num_input_tokens_seen': 4733272064, 'completed': '61.22% (2_257 / 3_687)', 'remaining time': '12:45:45', 'throughput': '8977.47', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 04:28:37,127 >> {'loss': 0.3236, 'grad_norm': 13.481849670410156, 'learning_rate': 4.2181253135072925e-07, 'epoch': 0.3954434499593165, 'num_input_tokens_seen': 4735369216, 'completed': '61.24% (2_258 / 3_687)', 'remaining time': '12:45:12', 'throughput': '8402.67', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 04:29:10,995 >> {'loss': 0.5844, 'grad_norm': 27.910301208496094, 'learning_rate': 4.21425619891302e-07, 'epoch': 0.3957146731760239, 'num_input_tokens_seen': 4737466368, 'completed': '61.27% (2_259 / 3_687)', 'remaining time': '12:44:42', 'throughput': '7740.32', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 04:29:43,357 >> {'loss': 0.6685, 'grad_norm': 24.167097091674805, 'learning_rate': 4.21038811903643e-07, 'epoch': 0.39598589639273124, 'num_input_tokens_seen': 4739563520, 'completed': '61.30% (2_260 / 3_687)', 'remaining time': '12:44:10', 'throughput': '8100.33', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 04:30:11,701 >> {'loss': 0.5551, 'grad_norm': 17.239904403686523, 'learning_rate': 4.206521076990409e-07, 'epoch': 0.39625711960943855, 'num_input_tokens_seen': 4741660672, 'completed': '61.32% (2_261 / 3_687)', 'remaining time': '12:43:34', 'throughput': '9248.42', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 04:30:43,434 >> {'loss': 0.4008, 'grad_norm': 21.12584114074707, 'learning_rate': 4.202655075887005e-07, 'epoch': 0.3965283428261459, 'num_input_tokens_seen': 4743757824, 'completed': '61.35% (2_262 / 3_687)', 'remaining time': '12:43:02', 'throughput': '8261.03', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 04:31:15,341 >> {'loss': 0.2911, 'grad_norm': 12.652347564697266, 'learning_rate': 4.1987901188374286e-07, 'epoch': 0.39679956604285327, 'num_input_tokens_seen': 4745854976, 'completed': '61.38% (2_263 / 3_687)', 'remaining time': '12:42:29', 'throughput': '8215.90', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 04:31:44,955 >> {'loss': 0.414, 'grad_norm': 13.351825714111328, 'learning_rate': 4.194926208952051e-07, 'epoch': 0.39707078925956063, 'num_input_tokens_seen': 4747952128, 'completed': '61.40% (2_264 / 3_687)', 'remaining time': '12:41:55', 'throughput': '8852.06', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 04:32:15,096 >> {'loss': 0.3764, 'grad_norm': 13.11346435546875, 'learning_rate': 4.191063349340397e-07, 'epoch': 0.397342012476268, 'num_input_tokens_seen': 4750049280, 'completed': '61.43% (2_265 / 3_687)', 'remaining time': '12:41:21', 'throughput': '8697.18', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 04:32:47,052 >> {'loss': 0.3352, 'grad_norm': 13.020018577575684, 'learning_rate': 4.1872015431111505e-07, 'epoch': 0.3976132356929753, 'num_input_tokens_seen': 4752146432, 'completed': '61.46% (2_266 / 3_687)', 'remaining time': '12:40:48', 'throughput': '8203.41', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 04:33:16,408 >> {'loss': 0.6615, 'grad_norm': 20.551406860351562, 'learning_rate': 4.1833407933721476e-07, 'epoch': 0.39788445890968266, 'num_input_tokens_seen': 4754243584, 'completed': '61.49% (2_267 / 3_687)', 'remaining time': '12:40:14', 'throughput': '8929.69', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 04:33:46,219 >> {'loss': 0.3388, 'grad_norm': 18.05826759338379, 'learning_rate': 4.17948110323037e-07, 'epoch': 0.39815568212639, 'num_input_tokens_seen': 4756340736, 'completed': '61.51% (2_268 / 3_687)', 'remaining time': '12:39:39', 'throughput': '8793.66', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 04:34:15,818 >> {'loss': 0.3536, 'grad_norm': 13.186254501342773, 'learning_rate': 4.1756224757919513e-07, 'epoch': 0.3984269053430974, 'num_input_tokens_seen': 4758437888, 'completed': '61.54% (2_269 / 3_687)', 'remaining time': '12:39:05', 'throughput': '8856.43', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 04:34:46,409 >> {'loss': 0.5714, 'grad_norm': 19.050817489624023, 'learning_rate': 4.17176491416217e-07, 'epoch': 0.39869812855980474, 'num_input_tokens_seen': 4760535040, 'completed': '61.57% (2_270 / 3_687)', 'remaining time': '12:38:31', 'throughput': '8569.22', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 04:35:16,984 >> {'loss': 0.2979, 'grad_norm': 13.171993255615234, 'learning_rate': 4.1679084214454405e-07, 'epoch': 0.39896935177651205, 'num_input_tokens_seen': 4762632192, 'completed': '61.59% (2_271 / 3_687)', 'remaining time': '12:37:57', 'throughput': '8573.97', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 04:35:47,890 >> {'loss': 0.4981, 'grad_norm': 19.698951721191406, 'learning_rate': 4.1640530007453245e-07, 'epoch': 0.3992405749932194, 'num_input_tokens_seen': 4764729344, 'completed': '61.62% (2_272 / 3_687)', 'remaining time': '12:37:24', 'throughput': '8481.81', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 04:36:19,170 >> {'loss': 0.3392, 'grad_norm': 14.195073127746582, 'learning_rate': 4.1601986551645163e-07, 'epoch': 0.39951179820992677, 'num_input_tokens_seen': 4766826496, 'completed': '61.65% (2_273 / 3_687)', 'remaining time': '12:36:51', 'throughput': '8380.56', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 04:36:49,154 >> {'loss': 0.3579, 'grad_norm': 14.913055419921875, 'learning_rate': 4.156345387804847e-07, 'epoch': 0.39978302142663413, 'num_input_tokens_seen': 4768923648, 'completed': '61.68% (2_274 / 3_687)', 'remaining time': '12:36:17', 'throughput': '8742.84', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 04:37:19,187 >> {'loss': 0.3177, 'grad_norm': 14.513870239257812, 'learning_rate': 4.152493201767281e-07, 'epoch': 0.4000542446433415, 'num_input_tokens_seen': 4771020800, 'completed': '61.70% (2_275 / 3_687)', 'remaining time': '12:35:43', 'throughput': '8728.65', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 04:37:47,568 >> {'loss': 0.6674, 'grad_norm': 18.92793083190918, 'learning_rate': 4.1486421001519087e-07, 'epoch': 0.4003254678600488, 'num_input_tokens_seen': 4773117952, 'completed': '61.73% (2_276 / 3_687)', 'remaining time': '12:35:07', 'throughput': '9236.60', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 04:38:19,214 >> {'loss': 0.3269, 'grad_norm': 14.322976112365723, 'learning_rate': 4.1447920860579524e-07, 'epoch': 0.40059669107675616, 'num_input_tokens_seen': 4775215104, 'completed': '61.76% (2_277 / 3_687)', 'remaining time': '12:34:35', 'throughput': '8283.46', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 04:38:47,058 >> {'loss': 0.5806, 'grad_norm': 20.151758193969727, 'learning_rate': 4.1409431625837545e-07, 'epoch': 0.4008679142934635, 'num_input_tokens_seen': 4777312256, 'completed': '61.78% (2_278 / 3_687)', 'remaining time': '12:33:59', 'throughput': '9414.85', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 04:39:20,372 >> {'loss': 0.3868, 'grad_norm': 17.131452560424805, 'learning_rate': 4.137095332826784e-07, 'epoch': 0.4011391375101709, 'num_input_tokens_seen': 4779409408, 'completed': '61.81% (2_279 / 3_687)', 'remaining time': '12:33:28', 'throughput': '7869.01', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 04:39:52,018 >> {'loss': 0.4183, 'grad_norm': 15.69462776184082, 'learning_rate': 4.1332485998836277e-07, 'epoch': 0.40141036072687825, 'num_input_tokens_seen': 4781506560, 'completed': '61.84% (2_280 / 3_687)', 'remaining time': '12:32:55', 'throughput': '8283.42', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 04:40:23,764 >> {'loss': 0.265, 'grad_norm': 13.978616714477539, 'learning_rate': 4.129402966849987e-07, 'epoch': 0.40168158394358555, 'num_input_tokens_seen': 4783603712, 'completed': '61.87% (2_281 / 3_687)', 'remaining time': '12:32:23', 'throughput': '8257.73', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 04:40:56,106 >> {'loss': 0.5206, 'grad_norm': 17.568675994873047, 'learning_rate': 4.1255584368206877e-07, 'epoch': 0.4019528071602929, 'num_input_tokens_seen': 4785700864, 'completed': '61.89% (2_282 / 3_687)', 'remaining time': '12:31:51', 'throughput': '8105.18', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 04:41:27,609 >> {'loss': 0.3761, 'grad_norm': 26.41534423828125, 'learning_rate': 4.121715012889655e-07, 'epoch': 0.4022240303770003, 'num_input_tokens_seen': 4787798016, 'completed': '61.92% (2_283 / 3_687)', 'remaining time': '12:31:18', 'throughput': '8321.28', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 04:41:57,080 >> {'loss': 0.3851, 'grad_norm': 14.635917663574219, 'learning_rate': 4.1178726981499313e-07, 'epoch': 0.40249525359370764, 'num_input_tokens_seen': 4789895168, 'completed': '61.95% (2_284 / 3_687)', 'remaining time': '12:30:43', 'throughput': '8895.11', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 04:42:27,397 >> {'loss': 0.6027, 'grad_norm': 18.9488582611084, 'learning_rate': 4.1140314956936663e-07, 'epoch': 0.402766476810415, 'num_input_tokens_seen': 4791992320, 'completed': '61.97% (2_285 / 3_687)', 'remaining time': '12:30:10', 'throughput': '8646.63', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 04:42:58,099 >> {'loss': 0.269, 'grad_norm': 17.327831268310547, 'learning_rate': 4.1101914086121137e-07, 'epoch': 0.4030377000271223, 'num_input_tokens_seen': 4794089472, 'completed': '62.00% (2_286 / 3_687)', 'remaining time': '12:29:36', 'throughput': '8538.48', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 04:43:29,354 >> {'loss': 0.4427, 'grad_norm': 15.44780445098877, 'learning_rate': 4.106352439995632e-07, 'epoch': 0.40330892324382966, 'num_input_tokens_seen': 4796186624, 'completed': '62.03% (2_287 / 3_687)', 'remaining time': '12:29:03', 'throughput': '8387.27', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 04:44:00,585 >> {'loss': 0.3452, 'grad_norm': 21.893918991088867, 'learning_rate': 4.102514592933671e-07, 'epoch': 0.403580146460537, 'num_input_tokens_seen': 4798283776, 'completed': '62.06% (2_288 / 3_687)', 'remaining time': '12:28:30', 'throughput': '8393.56', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 04:44:29,832 >> {'loss': 0.4563, 'grad_norm': 15.465867042541504, 'learning_rate': 4.098677870514788e-07, 'epoch': 0.4038513696772444, 'num_input_tokens_seen': 4800380928, 'completed': '62.08% (2_289 / 3_687)', 'remaining time': '12:27:56', 'throughput': '8963.29', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 04:45:00,750 >> {'loss': 0.4338, 'grad_norm': 18.96814727783203, 'learning_rate': 4.094842275826631e-07, 'epoch': 0.40412259289395175, 'num_input_tokens_seen': 4802478080, 'completed': '62.11% (2_290 / 3_687)', 'remaining time': '12:27:22', 'throughput': '8478.67', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 04:45:29,952 >> {'loss': 0.4135, 'grad_norm': 18.4038143157959, 'learning_rate': 4.091007811955941e-07, 'epoch': 0.40439381611065905, 'num_input_tokens_seen': 4804575232, 'completed': '62.14% (2_291 / 3_687)', 'remaining time': '12:26:48', 'throughput': '8976.74', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 04:45:57,221 >> {'loss': 0.5425, 'grad_norm': 16.829374313354492, 'learning_rate': 4.0871744819885467e-07, 'epoch': 0.4046650393273664, 'num_input_tokens_seen': 4806672384, 'completed': '62.16% (2_292 / 3_687)', 'remaining time': '12:26:11', 'throughput': '9613.51', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 04:46:28,722 >> {'loss': 0.5627, 'grad_norm': 16.508983612060547, 'learning_rate': 4.0833422890093684e-07, 'epoch': 0.4049362625440738, 'num_input_tokens_seen': 4808769536, 'completed': '62.19% (2_293 / 3_687)', 'remaining time': '12:25:38', 'throughput': '8321.73', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 04:47:01,343 >> {'loss': 0.5495, 'grad_norm': 19.47481918334961, 'learning_rate': 4.0795112361024075e-07, 'epoch': 0.40520748576078114, 'num_input_tokens_seen': 4810866688, 'completed': '62.22% (2_294 / 3_687)', 'remaining time': '12:25:07', 'throughput': '8035.99', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 04:47:32,672 >> {'loss': 0.3162, 'grad_norm': 12.910174369812012, 'learning_rate': 4.0756813263507496e-07, 'epoch': 0.4054787089774885, 'num_input_tokens_seen': 4812963840, 'completed': '62.25% (2_295 / 3_687)', 'remaining time': '12:24:34', 'throughput': '8367.49', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 04:48:05,325 >> {'loss': 0.3257, 'grad_norm': 15.603672981262207, 'learning_rate': 4.0718525628365617e-07, 'epoch': 0.4057499321941958, 'num_input_tokens_seen': 4815060992, 'completed': '62.27% (2_296 / 3_687)', 'remaining time': '12:24:02', 'throughput': '8028.22', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 04:48:36,149 >> {'loss': 0.4128, 'grad_norm': 17.36051368713379, 'learning_rate': 4.0680249486410845e-07, 'epoch': 0.40602115541090317, 'num_input_tokens_seen': 4817158144, 'completed': '62.30% (2_297 / 3_687)', 'remaining time': '12:23:29', 'throughput': '8504.36', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 04:49:07,695 >> {'loss': 0.5455, 'grad_norm': 20.466211318969727, 'learning_rate': 4.0641984868446386e-07, 'epoch': 0.40629237862761053, 'num_input_tokens_seen': 4819255296, 'completed': '62.33% (2_298 / 3_687)', 'remaining time': '12:22:56', 'throughput': '8309.88', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 04:49:40,828 >> {'loss': 0.3718, 'grad_norm': 15.06020736694336, 'learning_rate': 4.0603731805266175e-07, 'epoch': 0.4065636018443179, 'num_input_tokens_seen': 4821352448, 'completed': '62.35% (2_299 / 3_687)', 'remaining time': '12:22:25', 'throughput': '7911.94', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 04:50:12,104 >> {'loss': 0.4691, 'grad_norm': 16.064971923828125, 'learning_rate': 4.056549032765476e-07, 'epoch': 0.40683482506102525, 'num_input_tokens_seen': 4823449600, 'completed': '62.38% (2_300 / 3_687)', 'remaining time': '12:21:52', 'throughput': '8381.64', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 04:50:41,960 >> {'loss': 0.7604, 'grad_norm': 24.106367111206055, 'learning_rate': 4.0527260466387446e-07, 'epoch': 0.40710604827773256, 'num_input_tokens_seen': 4825546752, 'completed': '62.41% (2_301 / 3_687)', 'remaining time': '12:21:18', 'throughput': '8780.50', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 04:51:13,042 >> {'loss': 0.4701, 'grad_norm': 17.96955680847168, 'learning_rate': 4.0489042252230197e-07, 'epoch': 0.4073772714944399, 'num_input_tokens_seen': 4827643904, 'completed': '62.44% (2_302 / 3_687)', 'remaining time': '12:20:45', 'throughput': '8433.75', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 04:51:46,346 >> {'loss': 0.36, 'grad_norm': 13.112820625305176, 'learning_rate': 4.045083571593955e-07, 'epoch': 0.4076484947111473, 'num_input_tokens_seen': 4829741056, 'completed': '62.46% (2_303 / 3_687)', 'remaining time': '12:20:14', 'throughput': '7871.17', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 04:52:14,767 >> {'loss': 0.7148, 'grad_norm': 19.882753372192383, 'learning_rate': 4.0412640888262697e-07, 'epoch': 0.40791971792785464, 'num_input_tokens_seen': 4831838208, 'completed': '62.49% (2_304 / 3_687)', 'remaining time': '12:19:39', 'throughput': '9223.72', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 04:52:45,559 >> {'loss': 0.3871, 'grad_norm': 15.251856803894043, 'learning_rate': 4.0374457799937354e-07, 'epoch': 0.40819094114456195, 'num_input_tokens_seen': 4833935360, 'completed': '62.52% (2_305 / 3_687)', 'remaining time': '12:19:06', 'throughput': '8513.41', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 04:53:17,639 >> {'loss': 0.3569, 'grad_norm': 18.7488956451416, 'learning_rate': 4.033628648169184e-07, 'epoch': 0.4084621643612693, 'num_input_tokens_seen': 4836032512, 'completed': '62.54% (2_306 / 3_687)', 'remaining time': '12:18:33', 'throughput': '8171.66', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 04:53:47,086 >> {'loss': 0.561, 'grad_norm': 20.533279418945312, 'learning_rate': 4.0298126964244947e-07, 'epoch': 0.40873338757797667, 'num_input_tokens_seen': 4838129664, 'completed': '62.57% (2_307 / 3_687)', 'remaining time': '12:17:59', 'throughput': '8902.17', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 04:54:18,084 >> {'loss': 0.3199, 'grad_norm': 13.255075454711914, 'learning_rate': 4.025997927830604e-07, 'epoch': 0.40900461079468403, 'num_input_tokens_seen': 4840226816, 'completed': '62.60% (2_308 / 3_687)', 'remaining time': '12:17:26', 'throughput': '8456.75', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 04:54:48,352 >> {'loss': 0.3569, 'grad_norm': 18.75166130065918, 'learning_rate': 4.022184345457492e-07, 'epoch': 0.4092758340113914, 'num_input_tokens_seen': 4842323968, 'completed': '62.63% (2_309 / 3_687)', 'remaining time': '12:16:52', 'throughput': '8660.87', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 04:55:18,507 >> {'loss': 0.5254, 'grad_norm': 18.911184310913086, 'learning_rate': 4.0183719523741846e-07, 'epoch': 0.4095470572280987, 'num_input_tokens_seen': 4844421120, 'completed': '62.65% (2_310 / 3_687)', 'remaining time': '12:16:18', 'throughput': '8693.07', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 04:55:47,519 >> {'loss': 0.4653, 'grad_norm': 18.690418243408203, 'learning_rate': 4.014560751648747e-07, 'epoch': 0.40981828044480606, 'num_input_tokens_seen': 4846518272, 'completed': '62.68% (2_311 / 3_687)', 'remaining time': '12:15:43', 'throughput': '9035.88', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 04:56:15,015 >> {'loss': 0.4713, 'grad_norm': 19.980985641479492, 'learning_rate': 4.0107507463482924e-07, 'epoch': 0.4100895036615134, 'num_input_tokens_seen': 4848615424, 'completed': '62.71% (2_312 / 3_687)', 'remaining time': '12:15:07', 'throughput': '9533.87', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 04:56:44,981 >> {'loss': 0.5677, 'grad_norm': 18.602523803710938, 'learning_rate': 4.0069419395389657e-07, 'epoch': 0.4103607268782208, 'num_input_tokens_seen': 4850712576, 'completed': '62.73% (2_313 / 3_687)', 'remaining time': '12:14:33', 'throughput': '8747.99', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 04:57:14,941 >> {'loss': 0.4518, 'grad_norm': 14.569324493408203, 'learning_rate': 4.0031343342859526e-07, 'epoch': 0.41063195009492814, 'num_input_tokens_seen': 4852809728, 'completed': '62.76% (2_314 / 3_687)', 'remaining time': '12:13:59', 'throughput': '8749.65', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 04:57:44,329 >> {'loss': 0.5966, 'grad_norm': 17.369930267333984, 'learning_rate': 3.9993279336534667e-07, 'epoch': 0.41090317331163545, 'num_input_tokens_seen': 4854906880, 'completed': '62.79% (2_315 / 3_687)', 'remaining time': '12:13:25', 'throughput': '8920.19', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 04:58:15,078 >> {'loss': 0.568, 'grad_norm': 16.838441848754883, 'learning_rate': 3.995522740704758e-07, 'epoch': 0.4111743965283428, 'num_input_tokens_seen': 4857004032, 'completed': '62.82% (2_316 / 3_687)', 'remaining time': '12:12:51', 'throughput': '8525.25', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 04:58:48,215 >> {'loss': 0.3684, 'grad_norm': 16.305984497070312, 'learning_rate': 3.991718758502094e-07, 'epoch': 0.4114456197450502, 'num_input_tokens_seen': 4859101184, 'completed': '62.84% (2_317 / 3_687)', 'remaining time': '12:12:20', 'throughput': '7911.07', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 04:59:18,343 >> {'loss': 0.5516, 'grad_norm': 27.06982421875, 'learning_rate': 3.987915990106779e-07, 'epoch': 0.41171684296175753, 'num_input_tokens_seen': 4861198336, 'completed': '62.87% (2_318 / 3_687)', 'remaining time': '12:11:46', 'throughput': '8700.85', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 04:59:48,355 >> {'loss': 0.417, 'grad_norm': 15.275781631469727, 'learning_rate': 3.984114438579137e-07, 'epoch': 0.4119880661784649, 'num_input_tokens_seen': 4863295488, 'completed': '62.90% (2_319 / 3_687)', 'remaining time': '12:11:13', 'throughput': '8734.69', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 05:00:20,275 >> {'loss': 0.3962, 'grad_norm': 14.07772159576416, 'learning_rate': 3.980314106978512e-07, 'epoch': 0.4122592893951722, 'num_input_tokens_seen': 4865392640, 'completed': '62.92% (2_320 / 3_687)', 'remaining time': '12:10:40', 'throughput': '8212.41', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 05:00:50,009 >> {'loss': 0.3945, 'grad_norm': 14.107834815979004, 'learning_rate': 3.976514998363265e-07, 'epoch': 0.41253051261187956, 'num_input_tokens_seen': 4867489792, 'completed': '62.95% (2_321 / 3_687)', 'remaining time': '12:10:06', 'throughput': '8816.52', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 05:01:20,819 >> {'loss': 0.3535, 'grad_norm': 15.831769943237305, 'learning_rate': 3.972717115790773e-07, 'epoch': 0.4128017358285869, 'num_input_tokens_seen': 4869586944, 'completed': '62.98% (2_322 / 3_687)', 'remaining time': '12:09:33', 'throughput': '8508.21', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 05:01:50,939 >> {'loss': 0.4851, 'grad_norm': 16.84862518310547, 'learning_rate': 3.968920462317431e-07, 'epoch': 0.4130729590452943, 'num_input_tokens_seen': 4871684096, 'completed': '63.01% (2_323 / 3_687)', 'remaining time': '12:08:59', 'throughput': '8703.55', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 05:02:20,597 >> {'loss': 0.3473, 'grad_norm': 12.755271911621094, 'learning_rate': 3.965125040998637e-07, 'epoch': 0.41334418226200165, 'num_input_tokens_seen': 4873781248, 'completed': '63.03% (2_324 / 3_687)', 'remaining time': '12:08:25', 'throughput': '8838.89', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 05:02:48,657 >> {'loss': 0.4099, 'grad_norm': 15.923316955566406, 'learning_rate': 3.961330854888805e-07, 'epoch': 0.41361540547870895, 'num_input_tokens_seen': 4875878400, 'completed': '63.06% (2_325 / 3_687)', 'remaining time': '12:07:49', 'throughput': '9342.06', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 05:03:18,705 >> {'loss': 0.2539, 'grad_norm': 11.70615291595459, 'learning_rate': 3.9575379070413485e-07, 'epoch': 0.4138866286954163, 'num_input_tokens_seen': 4877975552, 'completed': '63.09% (2_326 / 3_687)', 'remaining time': '12:07:15', 'throughput': '8724.17', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 05:03:48,402 >> {'loss': 0.7379, 'grad_norm': 27.65726661682129, 'learning_rate': 3.953746200508693e-07, 'epoch': 0.4141578519121237, 'num_input_tokens_seen': 4880072704, 'completed': '63.11% (2_327 / 3_687)', 'remaining time': '12:06:41', 'throughput': '8827.35', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 05:04:17,427 >> {'loss': 0.7167, 'grad_norm': 20.938331604003906, 'learning_rate': 3.9499557383422534e-07, 'epoch': 0.41442907512883104, 'num_input_tokens_seen': 4882169856, 'completed': '63.14% (2_328 / 3_687)', 'remaining time': '12:06:06', 'throughput': '9031.79', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 05:04:46,610 >> {'loss': 0.2987, 'grad_norm': 13.526237487792969, 'learning_rate': 3.94616652359245e-07, 'epoch': 0.4147002983455384, 'num_input_tokens_seen': 4884267008, 'completed': '63.17% (2_329 / 3_687)', 'remaining time': '12:05:32', 'throughput': '8982.72', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 05:05:15,985 >> {'loss': 0.2497, 'grad_norm': 10.629281997680664, 'learning_rate': 3.942378559308703e-07, 'epoch': 0.4149715215622457, 'num_input_tokens_seen': 4886364160, 'completed': '63.20% (2_330 / 3_687)', 'remaining time': '12:04:57', 'throughput': '8923.85', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 05:05:48,585 >> {'loss': 0.3717, 'grad_norm': 15.359818458557129, 'learning_rate': 3.93859184853942e-07, 'epoch': 0.41524274477895307, 'num_input_tokens_seen': 4888461312, 'completed': '63.22% (2_331 / 3_687)', 'remaining time': '12:04:26', 'throughput': '8041.35', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 05:06:18,725 >> {'loss': 0.3598, 'grad_norm': 13.921889305114746, 'learning_rate': 3.934806394332001e-07, 'epoch': 0.4155139679956604, 'num_input_tokens_seen': 4890558464, 'completed': '63.25% (2_332 / 3_687)', 'remaining time': '12:03:52', 'throughput': '8697.44', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 05:06:48,906 >> {'loss': 0.5743, 'grad_norm': 21.2630558013916, 'learning_rate': 3.9310221997328363e-07, 'epoch': 0.4157851912123678, 'num_input_tokens_seen': 4892655616, 'completed': '63.28% (2_333 / 3_687)', 'remaining time': '12:03:18', 'throughput': '8685.83', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 05:07:18,981 >> {'loss': 0.6111, 'grad_norm': 20.155832290649414, 'learning_rate': 3.9272392677873e-07, 'epoch': 0.41605641442907515, 'num_input_tokens_seen': 4894752768, 'completed': '63.30% (2_334 / 3_687)', 'remaining time': '12:02:45', 'throughput': '8716.27', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 05:07:49,033 >> {'loss': 0.8151, 'grad_norm': 24.92179298400879, 'learning_rate': 3.9234576015397536e-07, 'epoch': 0.41632763764578246, 'num_input_tokens_seen': 4896849920, 'completed': '63.33% (2_335 / 3_687)', 'remaining time': '12:02:11', 'throughput': '8723.14', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 05:08:18,425 >> {'loss': 0.325, 'grad_norm': 14.339614868164062, 'learning_rate': 3.9196772040335367e-07, 'epoch': 0.4165988608624898, 'num_input_tokens_seen': 4898947072, 'completed': '63.36% (2_336 / 3_687)', 'remaining time': '12:01:36', 'throughput': '8918.82', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 05:08:48,056 >> {'loss': 0.6659, 'grad_norm': 24.353408813476562, 'learning_rate': 3.915898078310972e-07, 'epoch': 0.4168700840791972, 'num_input_tokens_seen': 4901044224, 'completed': '63.38% (2_337 / 3_687)', 'remaining time': '12:01:02', 'throughput': '8846.89', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 05:09:17,905 >> {'loss': 0.5632, 'grad_norm': 20.132877349853516, 'learning_rate': 3.9121202274133525e-07, 'epoch': 0.41714130729590454, 'num_input_tokens_seen': 4903141376, 'completed': '63.41% (2_338 / 3_687)', 'remaining time': '12:00:28', 'throughput': '8782.52', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 05:09:47,896 >> {'loss': 0.6138, 'grad_norm': 17.58623695373535, 'learning_rate': 3.9083436543809536e-07, 'epoch': 0.4174125305126119, 'num_input_tokens_seen': 4905238528, 'completed': '63.44% (2_339 / 3_687)', 'remaining time': '11:59:54', 'throughput': '8740.64', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 05:10:17,267 >> {'loss': 0.3483, 'grad_norm': 14.02330493927002, 'learning_rate': 3.904568362253011e-07, 'epoch': 0.4176837537293192, 'num_input_tokens_seen': 4907335680, 'completed': '63.47% (2_340 / 3_687)', 'remaining time': '11:59:20', 'throughput': '8925.33', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 05:10:46,695 >> {'loss': 0.4415, 'grad_norm': 15.86801528930664, 'learning_rate': 3.9007943540677426e-07, 'epoch': 0.41795497694602657, 'num_input_tokens_seen': 4909432832, 'completed': '63.49% (2_341 / 3_687)', 'remaining time': '11:58:46', 'throughput': '8908.00', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 05:11:17,362 >> {'loss': 0.3732, 'grad_norm': 14.43091869354248, 'learning_rate': 3.897021632862321e-07, 'epoch': 0.41822620016273393, 'num_input_tokens_seen': 4911529984, 'completed': '63.52% (2_342 / 3_687)', 'remaining time': '11:58:12', 'throughput': '8548.15', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 05:11:47,558 >> {'loss': 0.4415, 'grad_norm': 15.3654203414917, 'learning_rate': 3.893250201672893e-07, 'epoch': 0.4184974233794413, 'num_input_tokens_seen': 4913627136, 'completed': '63.55% (2_343 / 3_687)', 'remaining time': '11:57:39', 'throughput': '8681.16', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 05:12:20,789 >> {'loss': 0.4123, 'grad_norm': 15.816032409667969, 'learning_rate': 3.889480063534563e-07, 'epoch': 0.41876864659614865, 'num_input_tokens_seen': 4915724288, 'completed': '63.57% (2_344 / 3_687)', 'remaining time': '11:57:08', 'throughput': '7888.76', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 05:12:53,423 >> {'loss': 0.2871, 'grad_norm': 12.658620834350586, 'learning_rate': 3.88571122148139e-07, 'epoch': 0.41903986981285596, 'num_input_tokens_seen': 4917821440, 'completed': '63.60% (2_345 / 3_687)', 'remaining time': '11:56:36', 'throughput': '8032.75', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 05:13:25,600 >> {'loss': 0.4812, 'grad_norm': 14.271703720092773, 'learning_rate': 3.881943678546399e-07, 'epoch': 0.4193110930295633, 'num_input_tokens_seen': 4919918592, 'completed': '63.63% (2_346 / 3_687)', 'remaining time': '11:56:04', 'throughput': '8146.94', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 05:13:53,513 >> {'loss': 0.9452, 'grad_norm': 25.85860824584961, 'learning_rate': 3.878177437761564e-07, 'epoch': 0.4195823162462707, 'num_input_tokens_seen': 4922015744, 'completed': '63.66% (2_347 / 3_687)', 'remaining time': '11:55:29', 'throughput': '9391.30', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 05:14:20,384 >> {'loss': 0.47, 'grad_norm': 17.978992462158203, 'learning_rate': 3.8744125021578123e-07, 'epoch': 0.41985353946297804, 'num_input_tokens_seen': 4924112896, 'completed': '63.68% (2_348 / 3_687)', 'remaining time': '11:54:52', 'throughput': '9755.72', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 05:14:52,141 >> {'loss': 0.6245, 'grad_norm': 21.466154098510742, 'learning_rate': 3.87064887476502e-07, 'epoch': 0.4201247626796854, 'num_input_tokens_seen': 4926210048, 'completed': '63.71% (2_349 / 3_687)', 'remaining time': '11:54:20', 'throughput': '8254.79', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 05:15:22,943 >> {'loss': 0.3928, 'grad_norm': 17.588224411010742, 'learning_rate': 3.8668865586120124e-07, 'epoch': 0.4203959858963927, 'num_input_tokens_seen': 4928307200, 'completed': '63.74% (2_350 / 3_687)', 'remaining time': '11:53:47', 'throughput': '8510.42', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 05:15:54,613 >> {'loss': 0.5021, 'grad_norm': 19.852426528930664, 'learning_rate': 3.8631255567265573e-07, 'epoch': 0.42066720911310007, 'num_input_tokens_seen': 4930404352, 'completed': '63.76% (2_351 / 3_687)', 'remaining time': '11:53:15', 'throughput': '8277.52', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 05:16:25,633 >> {'loss': 0.3786, 'grad_norm': 17.130460739135742, 'learning_rate': 3.859365872135367e-07, 'epoch': 0.42093843232980743, 'num_input_tokens_seen': 4932501504, 'completed': '63.79% (2_352 / 3_687)', 'remaining time': '11:52:42', 'throughput': '8450.90', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 05:16:56,279 >> {'loss': 0.4072, 'grad_norm': 15.021489143371582, 'learning_rate': 3.8556075078640925e-07, 'epoch': 0.4212096555465148, 'num_input_tokens_seen': 4934598656, 'completed': '63.82% (2_353 / 3_687)', 'remaining time': '11:52:08', 'throughput': '8553.82', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 05:17:25,169 >> {'loss': 0.5125, 'grad_norm': 18.32381820678711, 'learning_rate': 3.85185046693732e-07, 'epoch': 0.42148087876322216, 'num_input_tokens_seen': 4936695808, 'completed': '63.85% (2_354 / 3_687)', 'remaining time': '11:51:34', 'throughput': '9073.89', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 05:17:55,054 >> {'loss': 0.4148, 'grad_norm': 14.185894966125488, 'learning_rate': 3.8480947523785767e-07, 'epoch': 0.42175210197992946, 'num_input_tokens_seen': 4938792960, 'completed': '63.87% (2_355 / 3_687)', 'remaining time': '11:51:00', 'throughput': '8771.70', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 05:18:25,535 >> {'loss': 0.497, 'grad_norm': 17.715118408203125, 'learning_rate': 3.844340367210318e-07, 'epoch': 0.4220233251966368, 'num_input_tokens_seen': 4940890112, 'completed': '63.90% (2_356 / 3_687)', 'remaining time': '11:50:26', 'throughput': '8600.37', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 05:18:58,080 >> {'loss': 0.3361, 'grad_norm': 13.911568641662598, 'learning_rate': 3.840587314453928e-07, 'epoch': 0.4222945484133442, 'num_input_tokens_seen': 4942987264, 'completed': '63.93% (2_357 / 3_687)', 'remaining time': '11:49:55', 'throughput': '8054.74', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 05:19:29,956 >> {'loss': 0.3143, 'grad_norm': 16.71425437927246, 'learning_rate': 3.8368355971297204e-07, 'epoch': 0.42256577163005155, 'num_input_tokens_seen': 4945084416, 'completed': '63.95% (2_358 / 3_687)', 'remaining time': '11:49:23', 'throughput': '8223.98', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 05:20:02,277 >> {'loss': 0.2808, 'grad_norm': 12.690278053283691, 'learning_rate': 3.8330852182569374e-07, 'epoch': 0.4228369948467589, 'num_input_tokens_seen': 4947181568, 'completed': '63.98% (2_359 / 3_687)', 'remaining time': '11:48:51', 'throughput': '8110.61', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 05:20:31,274 >> {'loss': 0.532, 'grad_norm': 17.643646240234375, 'learning_rate': 3.8293361808537404e-07, 'epoch': 0.4231082180634662, 'num_input_tokens_seen': 4949278720, 'completed': '64.01% (2_360 / 3_687)', 'remaining time': '11:48:16', 'throughput': '9040.24', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 05:21:00,195 >> {'loss': 0.5061, 'grad_norm': 19.790626525878906, 'learning_rate': 3.825588487937211e-07, 'epoch': 0.4233794412801736, 'num_input_tokens_seen': 4951375872, 'completed': '64.04% (2_361 / 3_687)', 'remaining time': '11:47:42', 'throughput': '9064.19', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 05:21:29,983 >> {'loss': 0.2747, 'grad_norm': 17.743762969970703, 'learning_rate': 3.821842142523352e-07, 'epoch': 0.42365066449688094, 'num_input_tokens_seen': 4953473024, 'completed': '64.06% (2_362 / 3_687)', 'remaining time': '11:47:08', 'throughput': '8800.27', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 05:22:01,516 >> {'loss': 0.4853, 'grad_norm': 18.460596084594727, 'learning_rate': 3.818097147627076e-07, 'epoch': 0.4239218877135883, 'num_input_tokens_seen': 4955570176, 'completed': '64.09% (2_363 / 3_687)', 'remaining time': '11:46:35', 'throughput': '8313.27', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 05:22:31,524 >> {'loss': 0.563, 'grad_norm': 20.445358276367188, 'learning_rate': 3.8143535062622154e-07, 'epoch': 0.42419311093029566, 'num_input_tokens_seen': 4957667328, 'completed': '64.12% (2_364 / 3_687)', 'remaining time': '11:46:02', 'throughput': '8735.92', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 05:23:01,675 >> {'loss': 0.6022, 'grad_norm': 21.970741271972656, 'learning_rate': 3.8106112214415087e-07, 'epoch': 0.42446433414700296, 'num_input_tokens_seen': 4959764480, 'completed': '64.14% (2_365 / 3_687)', 'remaining time': '11:45:28', 'throughput': '8694.22', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 05:23:32,295 >> {'loss': 0.4444, 'grad_norm': 15.904183387756348, 'learning_rate': 3.8068702961766053e-07, 'epoch': 0.4247355573637103, 'num_input_tokens_seen': 4961861632, 'completed': '64.17% (2_366 / 3_687)', 'remaining time': '11:44:55', 'throughput': '8561.29', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 05:24:03,108 >> {'loss': 0.1744, 'grad_norm': 9.053658485412598, 'learning_rate': 3.8031307334780594e-07, 'epoch': 0.4250067805804177, 'num_input_tokens_seen': 4963958784, 'completed': '64.20% (2_367 / 3_687)', 'remaining time': '11:44:22', 'throughput': '8507.66', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 05:24:32,903 >> {'loss': 0.3963, 'grad_norm': 14.599189758300781, 'learning_rate': 3.7993925363553294e-07, 'epoch': 0.42527800379712505, 'num_input_tokens_seen': 4966055936, 'completed': '64.23% (2_368 / 3_687)', 'remaining time': '11:43:48', 'throughput': '8798.35', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 05:25:05,423 >> {'loss': 0.5241, 'grad_norm': 18.069921493530273, 'learning_rate': 3.795655707816772e-07, 'epoch': 0.4255492270138324, 'num_input_tokens_seen': 4968153088, 'completed': '64.25% (2_369 / 3_687)', 'remaining time': '11:43:16', 'throughput': '8060.82', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 05:25:35,150 >> {'loss': 0.515, 'grad_norm': 19.06882095336914, 'learning_rate': 3.791920250869645e-07, 'epoch': 0.4258204502305397, 'num_input_tokens_seen': 4970250240, 'completed': '64.28% (2_370 / 3_687)', 'remaining time': '11:42:42', 'throughput': '8818.34', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 05:26:05,006 >> {'loss': 0.94, 'grad_norm': 26.823545455932617, 'learning_rate': 3.7881861685201033e-07, 'epoch': 0.4260916734472471, 'num_input_tokens_seen': 4972347392, 'completed': '64.31% (2_371 / 3_687)', 'remaining time': '11:42:09', 'throughput': '8780.42', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 05:26:33,888 >> {'loss': 0.2774, 'grad_norm': 12.676380157470703, 'learning_rate': 3.784453463773194e-07, 'epoch': 0.42636289666395444, 'num_input_tokens_seen': 4974444544, 'completed': '64.33% (2_372 / 3_687)', 'remaining time': '11:41:34', 'throughput': '9076.35', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 05:27:04,174 >> {'loss': 0.5369, 'grad_norm': 19.58039665222168, 'learning_rate': 3.780722139632858e-07, 'epoch': 0.4266341198806618, 'num_input_tokens_seen': 4976541696, 'completed': '64.36% (2_373 / 3_687)', 'remaining time': '11:41:00', 'throughput': '8655.76', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 05:27:32,575 >> {'loss': 0.7374, 'grad_norm': 23.31465721130371, 'learning_rate': 3.776992199101918e-07, 'epoch': 0.42690534309736916, 'num_input_tokens_seen': 4978638848, 'completed': '64.39% (2_374 / 3_687)', 'remaining time': '11:40:25', 'throughput': '9229.93', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 05:28:02,956 >> {'loss': 0.5331, 'grad_norm': 18.41512107849121, 'learning_rate': 3.773263645182091e-07, 'epoch': 0.42717656631407647, 'num_input_tokens_seen': 4980736000, 'completed': '64.42% (2_375 / 3_687)', 'remaining time': '11:39:52', 'throughput': '8628.48', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 05:28:34,765 >> {'loss': 0.3192, 'grad_norm': 15.421567916870117, 'learning_rate': 3.769536480873976e-07, 'epoch': 0.42744778953078383, 'num_input_tokens_seen': 4982833152, 'completed': '64.44% (2_376 / 3_687)', 'remaining time': '11:39:20', 'throughput': '8241.33', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 05:29:03,527 >> {'loss': 0.6033, 'grad_norm': 19.701927185058594, 'learning_rate': 3.765810709177052e-07, 'epoch': 0.4277190127474912, 'num_input_tokens_seen': 4984930304, 'completed': '64.47% (2_377 / 3_687)', 'remaining time': '11:38:45', 'throughput': '9114.17', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 05:29:32,064 >> {'loss': 0.6628, 'grad_norm': 20.730276107788086, 'learning_rate': 3.762086333089678e-07, 'epoch': 0.42799023596419855, 'num_input_tokens_seen': 4987027456, 'completed': '64.50% (2_378 / 3_687)', 'remaining time': '11:38:10', 'throughput': '9185.94', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 05:30:00,987 >> {'loss': 0.4378, 'grad_norm': 17.648303985595703, 'learning_rate': 3.758363355609092e-07, 'epoch': 0.4282614591809059, 'num_input_tokens_seen': 4989124608, 'completed': '64.52% (2_379 / 3_687)', 'remaining time': '11:37:36', 'throughput': '9063.65', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 05:30:29,919 >> {'loss': 0.3577, 'grad_norm': 14.124812126159668, 'learning_rate': 3.7546417797314023e-07, 'epoch': 0.4285326823976132, 'num_input_tokens_seen': 4991221760, 'completed': '64.55% (2_380 / 3_687)', 'remaining time': '11:37:01', 'throughput': '9060.72', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 05:30:57,834 >> {'loss': 0.4176, 'grad_norm': 16.618432998657227, 'learning_rate': 3.7509216084515916e-07, 'epoch': 0.4288039056143206, 'num_input_tokens_seen': 4993318912, 'completed': '64.58% (2_381 / 3_687)', 'remaining time': '11:36:26', 'throughput': '9390.74', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 05:31:27,067 >> {'loss': 0.553, 'grad_norm': 18.444599151611328, 'learning_rate': 3.747202844763514e-07, 'epoch': 0.42907512883102794, 'num_input_tokens_seen': 4995416064, 'completed': '64.61% (2_382 / 3_687)', 'remaining time': '11:35:52', 'throughput': '8967.44', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 05:31:56,539 >> {'loss': 0.6629, 'grad_norm': 21.762187957763672, 'learning_rate': 3.743485491659887e-07, 'epoch': 0.4293463520477353, 'num_input_tokens_seen': 4997513216, 'completed': '64.63% (2_383 / 3_687)', 'remaining time': '11:35:18', 'throughput': '8894.69', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 05:32:26,743 >> {'loss': 0.4474, 'grad_norm': 16.13730239868164, 'learning_rate': 3.739769552132298e-07, 'epoch': 0.42961757526444266, 'num_input_tokens_seen': 4999610368, 'completed': '64.66% (2_384 / 3_687)', 'remaining time': '11:34:44', 'throughput': '8679.24', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 05:32:59,749 >> {'loss': 0.2769, 'grad_norm': 12.197819709777832, 'learning_rate': 3.73605502917119e-07, 'epoch': 0.42988879848114997, 'num_input_tokens_seen': 5001707520, 'completed': '64.69% (2_385 / 3_687)', 'remaining time': '11:34:13', 'throughput': '7942.24', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 05:33:32,168 >> {'loss': 0.5748, 'grad_norm': 16.63471031188965, 'learning_rate': 3.7323419257658716e-07, 'epoch': 0.43016002169785733, 'num_input_tokens_seen': 5003804672, 'completed': '64.71% (2_386 / 3_687)', 'remaining time': '11:33:41', 'throughput': '8086.04', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 05:34:02,012 >> {'loss': 0.3293, 'grad_norm': 13.66933822631836, 'learning_rate': 3.728630244904507e-07, 'epoch': 0.4304312449145647, 'num_input_tokens_seen': 5005901824, 'completed': '64.74% (2_387 / 3_687)', 'remaining time': '11:33:08', 'throughput': '8783.95', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 05:34:32,970 >> {'loss': 0.4252, 'grad_norm': 15.115860939025879, 'learning_rate': 3.724919989574116e-07, 'epoch': 0.43070246813127205, 'num_input_tokens_seen': 5007998976, 'completed': '64.77% (2_388 / 3_687)', 'remaining time': '11:32:35', 'throughput': '8467.63', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 05:35:01,103 >> {'loss': 0.7401, 'grad_norm': 21.96157455444336, 'learning_rate': 3.7212111627605704e-07, 'epoch': 0.43097369134797936, 'num_input_tokens_seen': 5010096128, 'completed': '64.80% (2_389 / 3_687)', 'remaining time': '11:32:00', 'throughput': '9317.89', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 05:35:32,660 >> {'loss': 0.4966, 'grad_norm': 17.18273162841797, 'learning_rate': 3.717503767448593e-07, 'epoch': 0.4312449145646867, 'num_input_tokens_seen': 5012193280, 'completed': '64.82% (2_390 / 3_687)', 'remaining time': '11:31:27', 'throughput': '8307.12', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 05:36:02,650 >> {'loss': 0.4974, 'grad_norm': 23.42146110534668, 'learning_rate': 3.7137978066217555e-07, 'epoch': 0.4315161377813941, 'num_input_tokens_seen': 5014290432, 'completed': '64.85% (2_391 / 3_687)', 'remaining time': '11:30:54', 'throughput': '8741.10', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 05:36:32,995 >> {'loss': 0.7683, 'grad_norm': 21.250967025756836, 'learning_rate': 3.710093283262472e-07, 'epoch': 0.43178736099810144, 'num_input_tokens_seen': 5016387584, 'completed': '64.88% (2_392 / 3_687)', 'remaining time': '11:30:20', 'throughput': '8638.68', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 05:37:01,832 >> {'loss': 0.347, 'grad_norm': 14.832980155944824, 'learning_rate': 3.706390200352003e-07, 'epoch': 0.4320585842148088, 'num_input_tokens_seen': 5018484736, 'completed': '64.90% (2_393 / 3_687)', 'remaining time': '11:29:46', 'throughput': '9090.47', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 05:37:32,462 >> {'loss': 0.4751, 'grad_norm': 20.22023582458496, 'learning_rate': 3.7026885608704494e-07, 'epoch': 0.4323298074315161, 'num_input_tokens_seen': 5020581888, 'completed': '64.93% (2_394 / 3_687)', 'remaining time': '11:29:13', 'throughput': '8558.65', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 05:38:04,751 >> {'loss': 0.3332, 'grad_norm': 15.793986320495605, 'learning_rate': 3.6989883677967483e-07, 'epoch': 0.4326010306482235, 'num_input_tokens_seen': 5022679040, 'completed': '64.96% (2_395 / 3_687)', 'remaining time': '11:28:41', 'throughput': '8118.62', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 05:38:37,920 >> {'loss': 0.4048, 'grad_norm': 15.396735191345215, 'learning_rate': 3.6952896241086783e-07, 'epoch': 0.43287225386493083, 'num_input_tokens_seen': 5024776192, 'completed': '64.99% (2_396 / 3_687)', 'remaining time': '11:28:10', 'throughput': '7903.32', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 05:39:09,955 >> {'loss': 0.3603, 'grad_norm': 16.177085876464844, 'learning_rate': 3.6915923327828423e-07, 'epoch': 0.4331434770816382, 'num_input_tokens_seen': 5026873344, 'completed': '65.01% (2_397 / 3_687)', 'remaining time': '11:27:38', 'throughput': '8182.91', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 05:39:42,173 >> {'loss': 0.3433, 'grad_norm': 12.686070442199707, 'learning_rate': 3.6878964967946813e-07, 'epoch': 0.43341470029834556, 'num_input_tokens_seen': 5028970496, 'completed': '65.04% (2_398 / 3_687)', 'remaining time': '11:27:06', 'throughput': '8136.81', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 05:40:12,990 >> {'loss': 0.4353, 'grad_norm': 16.852243423461914, 'learning_rate': 3.6842021191184636e-07, 'epoch': 0.43368592351505286, 'num_input_tokens_seen': 5031067648, 'completed': '65.07% (2_399 / 3_687)', 'remaining time': '11:26:33', 'throughput': '8506.29', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 05:40:42,545 >> {'loss': 0.8005, 'grad_norm': 25.399932861328125, 'learning_rate': 3.6805092027272853e-07, 'epoch': 0.4339571467317602, 'num_input_tokens_seen': 5033164800, 'completed': '65.09% (2_400 / 3_687)', 'remaining time': '11:25:59', 'throughput': '8869.71', 'gpu_mem_free': '5581MB'} +/scratch3/workspace/ctpham_umass_edu-ft/envs/prolong-final/lib/python3.10/site-packages/torch/distributed/fsdp/fully_sharded_data_parallel.py:689: FutureWarning: FSDP.state_dict_type() and FSDP.set_state_dict_type() are being deprecated. Please use APIs, get_state_dict() and set_state_dict(), which can support different parallelisms, FSDP1, FSDP2, DDP. API doc: https://pytorch.org/docs/stable/distributed.checkpoint.html#torch.distributed.checkpoint.state_dict.get_state_dict .Tutorial: https://pytorch.org/tutorials/recipes/distributed_checkpoint_recipe.html . + warnings.warn( +[INFO|trainer.py:3503] 2024-12-26 05:41:05,933 >> Saving model checkpoint to /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/checkpoint-2400 +[INFO|configuration_utils.py:472] 2024-12-26 05:41:05,936 >> Configuration saved in /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/checkpoint-2400/config.json +[INFO|configuration_utils.py:807] 2024-12-26 05:41:05,938 >> Configuration saved in /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/checkpoint-2400/generation_config.json +[INFO|modeling_utils.py:2807] 2024-12-26 05:42:04,546 >> The model is bigger than the maximum size per checkpoint (5GB) and is going to be split in 7 checkpoint shards. You can find where each parameters has been saved in the index located at /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/checkpoint-2400/model.safetensors.index.json. +[INFO|tokenization_utils_base.py:2684] 2024-12-26 05:42:04,550 >> tokenizer config file saved in /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/checkpoint-2400/tokenizer_config.json +[INFO|tokenization_utils_base.py:2693] 2024-12-26 05:42:04,550 >> Special tokens file saved in /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/checkpoint-2400/special_tokens_map.json +/scratch3/workspace/ctpham_umass_edu-ft/envs/prolong-final/lib/python3.10/site-packages/torch/distributed/fsdp/fully_sharded_data_parallel.py:689: FutureWarning: FSDP.state_dict_type() and FSDP.set_state_dict_type() are being deprecated. Please use APIs, get_state_dict() and set_state_dict(), which can support different parallelisms, FSDP1, FSDP2, DDP. API doc: https://pytorch.org/docs/stable/distributed.checkpoint.html#torch.distributed.checkpoint.state_dict.get_state_dict .Tutorial: https://pytorch.org/tutorials/recipes/distributed_checkpoint_recipe.html . + warnings.warn( +[WARNING|trainer.py:868] 2024-12-26 05:45:49,977 >> Save streaming dataset state: {'epoch': 0, 'sample_in_epoch': 4800, 'num_canonical_nodes': 1, 'shuffle_seed': 42, 'initial_physical_nodes': 1} +/scratch3/workspace/ctpham_umass_edu-ft/envs/prolong-final/lib/python3.10/site-packages/torch/utils/checkpoint.py:1399: FutureWarning: `torch.cpu.amp.autocast(args...)` is deprecated. Please use `torch.amp.autocast('cpu', args...)` instead. + with device_autocast_ctx, torch.cpu.amp.autocast(**cpu_autocast_kwargs), recompute_context: # type: ignore[attr-defined] +[INFO|trainer.py:175] 2024-12-26 05:46:23,281 >> {'loss': 0.4201, 'grad_norm': 16.505773544311523, 'learning_rate': 3.6768177505930655e-07, 'epoch': 0.4342283699484676, 'num_input_tokens_seen': 5035261952, 'completed': '65.12% (2_401 / 3_687)', 'remaining time': '11:29:35', 'throughput': '769.35', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 05:46:55,393 >> {'loss': 0.5035, 'grad_norm': 18.550128936767578, 'learning_rate': 3.67312776568654e-07, 'epoch': 0.43449959316517495, 'num_input_tokens_seen': 5037359104, 'completed': '65.15% (2_402 / 3_687)', 'remaining time': '11:29:03', 'throughput': '8163.39', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 05:47:23,729 >> {'loss': 0.6078, 'grad_norm': 19.673357009887695, 'learning_rate': 3.669439250977272e-07, 'epoch': 0.4347708163818823, 'num_input_tokens_seen': 5039456256, 'completed': '65.17% (2_403 / 3_687)', 'remaining time': '11:28:28', 'throughput': '9251.23', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 05:47:52,824 >> {'loss': 0.5501, 'grad_norm': 17.886112213134766, 'learning_rate': 3.665752209433637e-07, 'epoch': 0.4350420395985896, 'num_input_tokens_seen': 5041553408, 'completed': '65.20% (2_404 / 3_687)', 'remaining time': '11:27:53', 'throughput': '9009.93', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 05:48:23,067 >> {'loss': 0.338, 'grad_norm': 13.1587495803833, 'learning_rate': 3.6620666440228254e-07, 'epoch': 0.435313262815297, 'num_input_tokens_seen': 5043650560, 'completed': '65.23% (2_405 / 3_687)', 'remaining time': '11:27:20', 'throughput': '8667.84', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 05:48:52,577 >> {'loss': 0.2726, 'grad_norm': 16.396703720092773, 'learning_rate': 3.6583825577108397e-07, 'epoch': 0.43558448603200434, 'num_input_tokens_seen': 5045747712, 'completed': '65.26% (2_406 / 3_687)', 'remaining time': '11:26:45', 'throughput': '8883.36', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 05:49:22,582 >> {'loss': 0.5378, 'grad_norm': 24.322254180908203, 'learning_rate': 3.654699953462494e-07, 'epoch': 0.4358557092487117, 'num_input_tokens_seen': 5047844864, 'completed': '65.28% (2_407 / 3_687)', 'remaining time': '11:26:11', 'throughput': '8736.65', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 05:49:52,389 >> {'loss': 0.333, 'grad_norm': 13.831670761108398, 'learning_rate': 3.651018834241406e-07, 'epoch': 0.43612693246541906, 'num_input_tokens_seen': 5049942016, 'completed': '65.31% (2_408 / 3_687)', 'remaining time': '11:25:37', 'throughput': '8794.70', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 05:50:26,276 >> {'loss': 0.3657, 'grad_norm': 13.609641075134277, 'learning_rate': 3.6473392030100014e-07, 'epoch': 0.43639815568212637, 'num_input_tokens_seen': 5052039168, 'completed': '65.34% (2_409 / 3_687)', 'remaining time': '11:25:06', 'throughput': '7735.88', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 05:50:57,757 >> {'loss': 0.4042, 'grad_norm': 15.657468795776367, 'learning_rate': 3.6436610627295074e-07, 'epoch': 0.4366693788988337, 'num_input_tokens_seen': 5054136320, 'completed': '65.36% (2_410 / 3_687)', 'remaining time': '11:24:34', 'throughput': '8326.93', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 05:51:25,847 >> {'loss': 0.6303, 'grad_norm': 18.23346519470215, 'learning_rate': 3.639984416359949e-07, 'epoch': 0.4369406021155411, 'num_input_tokens_seen': 5056233472, 'completed': '65.39% (2_411 / 3_687)', 'remaining time': '11:23:58', 'throughput': '9332.39', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 05:51:56,887 >> {'loss': 0.2942, 'grad_norm': 10.519538879394531, 'learning_rate': 3.636309266860156e-07, 'epoch': 0.43721182533224845, 'num_input_tokens_seen': 5058330624, 'completed': '65.42% (2_412 / 3_687)', 'remaining time': '11:23:25', 'throughput': '8445.36', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 05:52:26,795 >> {'loss': 0.5607, 'grad_norm': 17.80950927734375, 'learning_rate': 3.6326356171877483e-07, 'epoch': 0.4374830485489558, 'num_input_tokens_seen': 5060427776, 'completed': '65.45% (2_413 / 3_687)', 'remaining time': '11:22:51', 'throughput': '8765.03', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 05:52:59,400 >> {'loss': 0.348, 'grad_norm': 17.949668884277344, 'learning_rate': 3.6289634702991343e-07, 'epoch': 0.4377542717656631, 'num_input_tokens_seen': 5062524928, 'completed': '65.47% (2_414 / 3_687)', 'remaining time': '11:22:20', 'throughput': '8039.97', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 05:53:31,454 >> {'loss': 0.2518, 'grad_norm': 12.619994163513184, 'learning_rate': 3.625292829149521e-07, 'epoch': 0.4380254949823705, 'num_input_tokens_seen': 5064622080, 'completed': '65.50% (2_415 / 3_687)', 'remaining time': '11:21:47', 'throughput': '8178.23', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 05:54:01,385 >> {'loss': 0.785, 'grad_norm': 23.58609962463379, 'learning_rate': 3.6216236966929015e-07, 'epoch': 0.43829671819907784, 'num_input_tokens_seen': 5066719232, 'completed': '65.53% (2_416 / 3_687)', 'remaining time': '11:21:13', 'throughput': '8758.22', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 05:54:32,484 >> {'loss': 0.357, 'grad_norm': 17.113204956054688, 'learning_rate': 3.6179560758820527e-07, 'epoch': 0.4385679414157852, 'num_input_tokens_seen': 5068816384, 'completed': '65.55% (2_417 / 3_687)', 'remaining time': '11:20:40', 'throughput': '8429.37', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 05:55:02,465 >> {'loss': 0.6071, 'grad_norm': 29.91645622253418, 'learning_rate': 3.61428996966854e-07, 'epoch': 0.43883916463249256, 'num_input_tokens_seen': 5070913536, 'completed': '65.58% (2_418 / 3_687)', 'remaining time': '11:20:07', 'throughput': '8743.66', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 05:55:37,881 >> {'loss': 0.3174, 'grad_norm': 16.93893814086914, 'learning_rate': 3.610625381002701e-07, 'epoch': 0.43911038784919987, 'num_input_tokens_seen': 5073010688, 'completed': '65.61% (2_419 / 3_687)', 'remaining time': '11:19:37', 'throughput': '7401.88', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 05:56:08,360 >> {'loss': 0.5038, 'grad_norm': 17.732091903686523, 'learning_rate': 3.606962312833659e-07, 'epoch': 0.43938161106590723, 'num_input_tokens_seen': 5075107840, 'completed': '65.64% (2_420 / 3_687)', 'remaining time': '11:19:04', 'throughput': '8600.81', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 05:56:40,487 >> {'loss': 0.3963, 'grad_norm': 16.852190017700195, 'learning_rate': 3.603300768109311e-07, 'epoch': 0.4396528342826146, 'num_input_tokens_seen': 5077204992, 'completed': '65.66% (2_421 / 3_687)', 'remaining time': '11:18:31', 'throughput': '8159.55', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 05:57:11,479 >> {'loss': 0.67, 'grad_norm': 22.718647003173828, 'learning_rate': 3.5996407497763305e-07, 'epoch': 0.43992405749932195, 'num_input_tokens_seen': 5079302144, 'completed': '65.69% (2_422 / 3_687)', 'remaining time': '11:17:58', 'throughput': '8458.34', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 05:57:42,872 >> {'loss': 0.3855, 'grad_norm': 14.987257957458496, 'learning_rate': 3.5959822607801617e-07, 'epoch': 0.4401952807160293, 'num_input_tokens_seen': 5081399296, 'completed': '65.72% (2_423 / 3_687)', 'remaining time': '11:17:26', 'throughput': '8350.48', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 05:58:13,980 >> {'loss': 0.5356, 'grad_norm': 19.861112594604492, 'learning_rate': 3.592325304065018e-07, 'epoch': 0.4404665039327366, 'num_input_tokens_seen': 5083496448, 'completed': '65.74% (2_424 / 3_687)', 'remaining time': '11:16:53', 'throughput': '8426.93', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 05:58:43,550 >> {'loss': 0.4618, 'grad_norm': 17.07244873046875, 'learning_rate': 3.588669882573875e-07, 'epoch': 0.440737727149444, 'num_input_tokens_seen': 5085593600, 'completed': '65.77% (2_425 / 3_687)', 'remaining time': '11:16:18', 'throughput': '8865.13', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 05:59:12,283 >> {'loss': 0.9808, 'grad_norm': 27.226409912109375, 'learning_rate': 3.5850159992484787e-07, 'epoch': 0.44100895036615134, 'num_input_tokens_seen': 5087690752, 'completed': '65.80% (2_426 / 3_687)', 'remaining time': '11:15:44', 'throughput': '9123.56', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 05:59:43,183 >> {'loss': 0.344, 'grad_norm': 16.227130889892578, 'learning_rate': 3.581363657029336e-07, 'epoch': 0.4412801735828587, 'num_input_tokens_seen': 5089787904, 'completed': '65.83% (2_427 / 3_687)', 'remaining time': '11:15:10', 'throughput': '8483.48', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 06:00:16,044 >> {'loss': 0.202, 'grad_norm': 9.734285354614258, 'learning_rate': 3.5777128588557126e-07, 'epoch': 0.44155139679956606, 'num_input_tokens_seen': 5091885056, 'completed': '65.85% (2_428 / 3_687)', 'remaining time': '11:14:39', 'throughput': '7977.56', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 06:00:47,086 >> {'loss': 0.6473, 'grad_norm': 18.710073471069336, 'learning_rate': 3.574063607665633e-07, 'epoch': 0.44182262001627337, 'num_input_tokens_seen': 5093982208, 'completed': '65.88% (2_429 / 3_687)', 'remaining time': '11:14:06', 'throughput': '8444.74', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 06:01:16,563 >> {'loss': 0.5162, 'grad_norm': 18.220901489257812, 'learning_rate': 3.570415906395873e-07, 'epoch': 0.44209384323298073, 'num_input_tokens_seen': 5096079360, 'completed': '65.91% (2_430 / 3_687)', 'remaining time': '11:13:32', 'throughput': '8893.14', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 06:01:46,517 >> {'loss': 0.4738, 'grad_norm': 17.251747131347656, 'learning_rate': 3.5667697579819655e-07, 'epoch': 0.4423650664496881, 'num_input_tokens_seen': 5098176512, 'completed': '65.93% (2_431 / 3_687)', 'remaining time': '11:12:58', 'throughput': '8751.57', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 06:02:16,919 >> {'loss': 0.5164, 'grad_norm': 19.4804744720459, 'learning_rate': 3.563125165358193e-07, 'epoch': 0.44263628966639545, 'num_input_tokens_seen': 5100273664, 'completed': '65.96% (2_432 / 3_687)', 'remaining time': '11:12:24', 'throughput': '8622.54', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 06:02:46,297 >> {'loss': 0.4864, 'grad_norm': 17.455604553222656, 'learning_rate': 3.559482131457583e-07, 'epoch': 0.4429075128831028, 'num_input_tokens_seen': 5102370816, 'completed': '65.99% (2_433 / 3_687)', 'remaining time': '11:11:50', 'throughput': '8923.21', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 06:03:17,921 >> {'loss': 0.4475, 'grad_norm': 16.524316787719727, 'learning_rate': 3.5558406592119115e-07, 'epoch': 0.4431787360998101, 'num_input_tokens_seen': 5104467968, 'completed': '66.02% (2_434 / 3_687)', 'remaining time': '11:11:17', 'throughput': '8289.32', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 06:03:47,096 >> {'loss': 0.5127, 'grad_norm': 15.812894821166992, 'learning_rate': 3.552200751551697e-07, 'epoch': 0.4434499593165175, 'num_input_tokens_seen': 5106565120, 'completed': '66.04% (2_435 / 3_687)', 'remaining time': '11:10:43', 'throughput': '8985.19', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 06:04:17,048 >> {'loss': 0.4391, 'grad_norm': 17.802688598632812, 'learning_rate': 3.548562411406201e-07, 'epoch': 0.44372118253322484, 'num_input_tokens_seen': 5108662272, 'completed': '66.07% (2_436 / 3_687)', 'remaining time': '11:10:09', 'throughput': '8752.11', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 06:04:47,664 >> {'loss': 0.465, 'grad_norm': 21.305082321166992, 'learning_rate': 3.544925641703413e-07, 'epoch': 0.4439924057499322, 'num_input_tokens_seen': 5110759424, 'completed': '66.10% (2_437 / 3_687)', 'remaining time': '11:09:36', 'throughput': '8562.38', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 06:05:16,343 >> {'loss': 0.6903, 'grad_norm': 23.760269165039062, 'learning_rate': 3.5412904453700754e-07, 'epoch': 0.44426362896663957, 'num_input_tokens_seen': 5112856576, 'completed': '66.12% (2_438 / 3_687)', 'remaining time': '11:09:01', 'throughput': '9140.56', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 06:05:44,450 >> {'loss': 0.6868, 'grad_norm': 35.48469543457031, 'learning_rate': 3.537656825331653e-07, 'epoch': 0.4445348521833469, 'num_input_tokens_seen': 5114953728, 'completed': '66.15% (2_439 / 3_687)', 'remaining time': '11:08:26', 'throughput': '9326.67', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 06:06:17,643 >> {'loss': 0.5361, 'grad_norm': 18.90154457092285, 'learning_rate': 3.534024784512345e-07, 'epoch': 0.44480607540005423, 'num_input_tokens_seen': 5117050880, 'completed': '66.18% (2_440 / 3_687)', 'remaining time': '11:07:55', 'throughput': '7897.63', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 06:06:48,821 >> {'loss': 0.4076, 'grad_norm': 16.816068649291992, 'learning_rate': 3.5303943258350813e-07, 'epoch': 0.4450772986167616, 'num_input_tokens_seen': 5119148032, 'completed': '66.21% (2_441 / 3_687)', 'remaining time': '11:07:22', 'throughput': '8408.03', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 06:07:20,078 >> {'loss': 0.3967, 'grad_norm': 31.906917572021484, 'learning_rate': 3.526765452221512e-07, 'epoch': 0.44534852183346896, 'num_input_tokens_seen': 5121245184, 'completed': '66.23% (2_442 / 3_687)', 'remaining time': '11:06:49', 'throughput': '8386.59', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 06:07:52,329 >> {'loss': 0.468, 'grad_norm': 16.148090362548828, 'learning_rate': 3.523138166592021e-07, 'epoch': 0.4456197450501763, 'num_input_tokens_seen': 5123342336, 'completed': '66.26% (2_443 / 3_687)', 'remaining time': '11:06:17', 'throughput': '8128.36', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 06:08:22,275 >> {'loss': 0.7676, 'grad_norm': 22.877065658569336, 'learning_rate': 3.5195124718657075e-07, 'epoch': 0.4458909682668836, 'num_input_tokens_seen': 5125439488, 'completed': '66.29% (2_444 / 3_687)', 'remaining time': '11:05:43', 'throughput': '8753.84', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 06:08:52,076 >> {'loss': 0.3127, 'grad_norm': 16.407276153564453, 'learning_rate': 3.5158883709603946e-07, 'epoch': 0.446162191483591, 'num_input_tokens_seen': 5127536640, 'completed': '66.31% (2_445 / 3_687)', 'remaining time': '11:05:09', 'throughput': '8796.64', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 06:09:26,225 >> {'loss': 0.5557, 'grad_norm': 17.211170196533203, 'learning_rate': 3.5122658667926177e-07, 'epoch': 0.44643341470029835, 'num_input_tokens_seen': 5129633792, 'completed': '66.34% (2_446 / 3_687)', 'remaining time': '11:04:39', 'throughput': '7676.36', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 06:09:55,282 >> {'loss': 0.6258, 'grad_norm': 17.69900131225586, 'learning_rate': 3.5086449622776346e-07, 'epoch': 0.4467046379170057, 'num_input_tokens_seen': 5131730944, 'completed': '66.37% (2_447 / 3_687)', 'remaining time': '11:04:04', 'throughput': '9021.58', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 06:10:25,859 >> {'loss': 0.441, 'grad_norm': 17.46190643310547, 'learning_rate': 3.505025660329408e-07, 'epoch': 0.44697586113371307, 'num_input_tokens_seen': 5133828096, 'completed': '66.40% (2_448 / 3_687)', 'remaining time': '11:03:31', 'throughput': '8573.41', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 06:10:56,341 >> {'loss': 0.3493, 'grad_norm': 13.633960723876953, 'learning_rate': 3.5014079638606164e-07, 'epoch': 0.4472470843504204, 'num_input_tokens_seen': 5135925248, 'completed': '66.42% (2_449 / 3_687)', 'remaining time': '11:02:57', 'throughput': '8599.99', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 06:11:24,726 >> {'loss': 0.3263, 'grad_norm': 15.675495147705078, 'learning_rate': 3.497791875782643e-07, 'epoch': 0.44751830756712774, 'num_input_tokens_seen': 5138022400, 'completed': '66.45% (2_450 / 3_687)', 'remaining time': '11:02:23', 'throughput': '9235.09', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 06:11:56,370 >> {'loss': 0.4154, 'grad_norm': 17.797544479370117, 'learning_rate': 3.4941773990055777e-07, 'epoch': 0.4477895307838351, 'num_input_tokens_seen': 5140119552, 'completed': '66.48% (2_451 / 3_687)', 'remaining time': '11:01:50', 'throughput': '8284.30', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 06:12:25,946 >> {'loss': 0.7703, 'grad_norm': 26.41427993774414, 'learning_rate': 3.490564536438215e-07, 'epoch': 0.44806075400054246, 'num_input_tokens_seen': 5142216704, 'completed': '66.50% (2_452 / 3_687)', 'remaining time': '11:01:16', 'throughput': '8863.30', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 06:12:55,931 >> {'loss': 0.4778, 'grad_norm': 16.2842960357666, 'learning_rate': 3.4869532909880485e-07, 'epoch': 0.4483319772172498, 'num_input_tokens_seen': 5144313856, 'completed': '66.53% (2_453 / 3_687)', 'remaining time': '11:00:42', 'throughput': '8742.43', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 06:13:25,489 >> {'loss': 0.3942, 'grad_norm': 18.36961555480957, 'learning_rate': 3.483343665561271e-07, 'epoch': 0.4486032004339571, 'num_input_tokens_seen': 5146411008, 'completed': '66.56% (2_454 / 3_687)', 'remaining time': '11:00:08', 'throughput': '8869.00', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 06:13:58,058 >> {'loss': 0.5367, 'grad_norm': 19.449003219604492, 'learning_rate': 3.479735663062773e-07, 'epoch': 0.4488744236506645, 'num_input_tokens_seen': 5148508160, 'completed': '66.59% (2_455 / 3_687)', 'remaining time': '10:59:36', 'throughput': '8048.99', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 06:14:27,951 >> {'loss': 0.3392, 'grad_norm': 11.512831687927246, 'learning_rate': 3.4761292863961354e-07, 'epoch': 0.44914564686737185, 'num_input_tokens_seen': 5150605312, 'completed': '66.61% (2_456 / 3_687)', 'remaining time': '10:59:03', 'throughput': '8769.10', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 06:14:58,717 >> {'loss': 0.4308, 'grad_norm': 17.354673385620117, 'learning_rate': 3.4725245384636347e-07, 'epoch': 0.4494168700840792, 'num_input_tokens_seen': 5152702464, 'completed': '66.64% (2_457 / 3_687)', 'remaining time': '10:58:30', 'throughput': '8520.69', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 06:15:28,938 >> {'loss': 0.2647, 'grad_norm': 12.854970932006836, 'learning_rate': 3.4689214221662364e-07, 'epoch': 0.4496880933007866, 'num_input_tokens_seen': 5154799616, 'completed': '66.67% (2_458 / 3_687)', 'remaining time': '10:57:56', 'throughput': '8674.13', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 06:15:59,875 >> {'loss': 0.5143, 'grad_norm': 16.739013671875, 'learning_rate': 3.465319940403587e-07, 'epoch': 0.4499593165174939, 'num_input_tokens_seen': 5156896768, 'completed': '66.69% (2_459 / 3_687)', 'remaining time': '10:57:23', 'throughput': '8473.57', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 06:16:29,558 >> {'loss': 0.3709, 'grad_norm': 15.290994644165039, 'learning_rate': 3.4617200960740247e-07, 'epoch': 0.45023053973420124, 'num_input_tokens_seen': 5158993920, 'completed': '66.72% (2_460 / 3_687)', 'remaining time': '10:56:49', 'throughput': '8831.52', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 06:17:01,243 >> {'loss': 0.34, 'grad_norm': 12.358589172363281, 'learning_rate': 3.4581218920745663e-07, 'epoch': 0.4505017629509086, 'num_input_tokens_seen': 5161091072, 'completed': '66.75% (2_461 / 3_687)', 'remaining time': '10:56:17', 'throughput': '8273.30', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 06:17:30,717 >> {'loss': 0.3167, 'grad_norm': 13.680041313171387, 'learning_rate': 3.454525331300908e-07, 'epoch': 0.45077298616761596, 'num_input_tokens_seen': 5163188224, 'completed': '66.78% (2_462 / 3_687)', 'remaining time': '10:55:43', 'throughput': '8894.03', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 06:18:01,850 >> {'loss': 0.4137, 'grad_norm': 22.562665939331055, 'learning_rate': 3.450930416647429e-07, 'epoch': 0.4510442093843233, 'num_input_tokens_seen': 5165285376, 'completed': '66.80% (2_463 / 3_687)', 'remaining time': '10:55:10', 'throughput': '8420.30', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 06:18:31,592 >> {'loss': 0.4412, 'grad_norm': 14.763349533081055, 'learning_rate': 3.4473371510071795e-07, 'epoch': 0.45131543260103063, 'num_input_tokens_seen': 5167382528, 'completed': '66.83% (2_464 / 3_687)', 'remaining time': '10:54:36', 'throughput': '8813.78', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 06:19:02,750 >> {'loss': 0.578, 'grad_norm': 20.03864097595215, 'learning_rate': 3.4437455372718795e-07, 'epoch': 0.451586655817738, 'num_input_tokens_seen': 5169479680, 'completed': '66.86% (2_465 / 3_687)', 'remaining time': '10:54:03', 'throughput': '8413.46', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 06:19:35,576 >> {'loss': 0.3843, 'grad_norm': 19.433835983276367, 'learning_rate': 3.440155578331925e-07, 'epoch': 0.45185787903444535, 'num_input_tokens_seen': 5171576832, 'completed': '66.88% (2_466 / 3_687)', 'remaining time': '10:53:31', 'throughput': '7985.84', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 06:20:03,064 >> {'loss': 0.3445, 'grad_norm': 15.715560913085938, 'learning_rate': 3.4365672770763783e-07, 'epoch': 0.4521291022511527, 'num_input_tokens_seen': 5173673984, 'completed': '66.91% (2_467 / 3_687)', 'remaining time': '10:52:56', 'throughput': '9536.69', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 06:20:35,602 >> {'loss': 0.3954, 'grad_norm': 16.51280975341797, 'learning_rate': 3.432980636392967e-07, 'epoch': 0.4524003254678601, 'num_input_tokens_seen': 5175771136, 'completed': '66.94% (2_468 / 3_687)', 'remaining time': '10:52:24', 'throughput': '8056.50', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 06:21:05,524 >> {'loss': 0.3528, 'grad_norm': 13.621045112609863, 'learning_rate': 3.429395659168084e-07, 'epoch': 0.4526715486845674, 'num_input_tokens_seen': 5177868288, 'completed': '66.97% (2_469 / 3_687)', 'remaining time': '10:51:50', 'throughput': '8761.03', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 06:21:33,663 >> {'loss': 0.5579, 'grad_norm': 19.30001449584961, 'learning_rate': 3.425812348286782e-07, 'epoch': 0.45294277190127474, 'num_input_tokens_seen': 5179965440, 'completed': '66.99% (2_470 / 3_687)', 'remaining time': '10:51:15', 'throughput': '9316.10', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 06:22:01,272 >> {'loss': 0.8619, 'grad_norm': 25.89718246459961, 'learning_rate': 3.422230706632774e-07, 'epoch': 0.4532139951179821, 'num_input_tokens_seen': 5182062592, 'completed': '67.02% (2_471 / 3_687)', 'remaining time': '10:50:40', 'throughput': '9494.76', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 06:22:30,973 >> {'loss': 0.489, 'grad_norm': 17.709318161010742, 'learning_rate': 3.418650737088427e-07, 'epoch': 0.45348521833468947, 'num_input_tokens_seen': 5184159744, 'completed': '67.05% (2_472 / 3_687)', 'remaining time': '10:50:06', 'throughput': '8826.00', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 06:23:02,231 >> {'loss': 0.3265, 'grad_norm': 17.07084846496582, 'learning_rate': 3.415072442534767e-07, 'epoch': 0.45375644155139677, 'num_input_tokens_seen': 5186256896, 'completed': '67.07% (2_473 / 3_687)', 'remaining time': '10:49:33', 'throughput': '8386.59', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 06:23:33,564 >> {'loss': 0.3484, 'grad_norm': 14.670482635498047, 'learning_rate': 3.411495825851467e-07, 'epoch': 0.45402766476810413, 'num_input_tokens_seen': 5188354048, 'completed': '67.10% (2_474 / 3_687)', 'remaining time': '10:49:01', 'throughput': '8366.37', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 06:24:03,273 >> {'loss': 0.3738, 'grad_norm': 15.770243644714355, 'learning_rate': 3.4079208899168545e-07, 'epoch': 0.4542988879848115, 'num_input_tokens_seen': 5190451200, 'completed': '67.13% (2_475 / 3_687)', 'remaining time': '10:48:27', 'throughput': '8823.78', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 06:24:34,149 >> {'loss': 0.4085, 'grad_norm': 18.940078735351562, 'learning_rate': 3.404347637607899e-07, 'epoch': 0.45457011120151886, 'num_input_tokens_seen': 5192548352, 'completed': '67.15% (2_476 / 3_687)', 'remaining time': '10:47:54', 'throughput': '8490.06', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 06:25:03,644 >> {'loss': 0.5006, 'grad_norm': 20.297151565551758, 'learning_rate': 3.400776071800219e-07, 'epoch': 0.4548413344182262, 'num_input_tokens_seen': 5194645504, 'completed': '67.18% (2_477 / 3_687)', 'remaining time': '10:47:20', 'throughput': '8887.86', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 06:25:33,558 >> {'loss': 0.2718, 'grad_norm': 13.716742515563965, 'learning_rate': 3.3972061953680734e-07, 'epoch': 0.4551125576349335, 'num_input_tokens_seen': 5196742656, 'completed': '67.21% (2_478 / 3_687)', 'remaining time': '10:46:46', 'throughput': '8763.22', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 06:26:06,626 >> {'loss': 0.4441, 'grad_norm': 16.452495574951172, 'learning_rate': 3.3936380111843666e-07, 'epoch': 0.4553837808516409, 'num_input_tokens_seen': 5198839808, 'completed': '67.24% (2_479 / 3_687)', 'remaining time': '10:46:15', 'throughput': '7927.52', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 06:26:38,628 >> {'loss': 0.3707, 'grad_norm': 12.949135780334473, 'learning_rate': 3.390071522120635e-07, 'epoch': 0.45565500406834825, 'num_input_tokens_seen': 5200936960, 'completed': '67.26% (2_480 / 3_687)', 'remaining time': '10:45:43', 'throughput': '8191.45', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 06:27:09,762 >> {'loss': 0.3601, 'grad_norm': 13.870152473449707, 'learning_rate': 3.3865067310470554e-07, 'epoch': 0.4559262272850556, 'num_input_tokens_seen': 5203034112, 'completed': '67.29% (2_481 / 3_687)', 'remaining time': '10:45:10', 'throughput': '8419.76', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 06:27:38,877 >> {'loss': 0.3517, 'grad_norm': 12.890467643737793, 'learning_rate': 3.3829436408324316e-07, 'epoch': 0.45619745050176297, 'num_input_tokens_seen': 5205131264, 'completed': '67.32% (2_482 / 3_687)', 'remaining time': '10:44:36', 'throughput': '9003.89', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 06:28:10,520 >> {'loss': 0.4483, 'grad_norm': 14.944725036621094, 'learning_rate': 3.3793822543442074e-07, 'epoch': 0.4564686737184703, 'num_input_tokens_seen': 5207228416, 'completed': '67.34% (2_483 / 3_687)', 'remaining time': '10:44:03', 'throughput': '8284.34', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 06:28:43,383 >> {'loss': 0.4053, 'grad_norm': 16.919103622436523, 'learning_rate': 3.3758225744484483e-07, 'epoch': 0.45673989693517764, 'num_input_tokens_seen': 5209325568, 'completed': '67.37% (2_484 / 3_687)', 'remaining time': '10:43:32', 'throughput': '7976.92', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 06:29:15,376 >> {'loss': 0.4156, 'grad_norm': 14.759577751159668, 'learning_rate': 3.372264604009851e-07, 'epoch': 0.457011120151885, 'num_input_tokens_seen': 5211422720, 'completed': '67.40% (2_485 / 3_687)', 'remaining time': '10:43:00', 'throughput': '8193.78', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 06:29:44,470 >> {'loss': 0.4126, 'grad_norm': 15.936949729919434, 'learning_rate': 3.3687083458917344e-07, 'epoch': 0.45728234336859236, 'num_input_tokens_seen': 5213519872, 'completed': '67.43% (2_486 / 3_687)', 'remaining time': '10:42:25', 'throughput': '9010.12', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 06:30:15,661 >> {'loss': 0.3871, 'grad_norm': 15.409136772155762, 'learning_rate': 3.3651538029560377e-07, 'epoch': 0.4575535665852997, 'num_input_tokens_seen': 5215617024, 'completed': '67.45% (2_487 / 3_687)', 'remaining time': '10:41:53', 'throughput': '8404.56', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 06:30:44,282 >> {'loss': 0.4349, 'grad_norm': 15.34483814239502, 'learning_rate': 3.361600978063325e-07, 'epoch': 0.457824789802007, 'num_input_tokens_seen': 5217714176, 'completed': '67.48% (2_488 / 3_687)', 'remaining time': '10:41:18', 'throughput': '9159.20', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 06:31:13,890 >> {'loss': 0.4529, 'grad_norm': 18.0634708404541, 'learning_rate': 3.358049874072771e-07, 'epoch': 0.4580960130187144, 'num_input_tokens_seen': 5219811328, 'completed': '67.51% (2_489 / 3_687)', 'remaining time': '10:40:44', 'throughput': '8853.71', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 06:31:45,525 >> {'loss': 0.4033, 'grad_norm': 17.72582244873047, 'learning_rate': 3.3545004938421734e-07, 'epoch': 0.45836723623542175, 'num_input_tokens_seen': 5221908480, 'completed': '67.53% (2_490 / 3_687)', 'remaining time': '10:40:12', 'throughput': '8286.47', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 06:32:12,877 >> {'loss': 0.7801, 'grad_norm': 20.901613235473633, 'learning_rate': 3.3509528402279357e-07, 'epoch': 0.4586384594521291, 'num_input_tokens_seen': 5224005632, 'completed': '67.56% (2_491 / 3_687)', 'remaining time': '10:39:36', 'throughput': '9584.21', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 06:32:44,377 >> {'loss': 0.9615, 'grad_norm': 34.06769943237305, 'learning_rate': 3.347406916085074e-07, 'epoch': 0.45890968266883647, 'num_input_tokens_seen': 5226102784, 'completed': '67.59% (2_492 / 3_687)', 'remaining time': '10:39:04', 'throughput': '8322.01', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 06:33:15,591 >> {'loss': 0.7134, 'grad_norm': 19.558860778808594, 'learning_rate': 3.3438627242672164e-07, 'epoch': 0.4591809058855438, 'num_input_tokens_seen': 5228199936, 'completed': '67.62% (2_493 / 3_687)', 'remaining time': '10:38:31', 'throughput': '8398.40', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 06:33:46,832 >> {'loss': 0.4235, 'grad_norm': 17.086078643798828, 'learning_rate': 3.3403202676265875e-07, 'epoch': 0.45945212910225114, 'num_input_tokens_seen': 5230297088, 'completed': '67.64% (2_494 / 3_687)', 'remaining time': '10:37:58', 'throughput': '8390.88', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 06:34:20,162 >> {'loss': 0.2654, 'grad_norm': 11.597755432128906, 'learning_rate': 3.336779549014026e-07, 'epoch': 0.4597233523189585, 'num_input_tokens_seen': 5232394240, 'completed': '67.67% (2_495 / 3_687)', 'remaining time': '10:37:27', 'throughput': '7865.09', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 06:34:49,598 >> {'loss': 0.3075, 'grad_norm': 12.856144905090332, 'learning_rate': 3.333240571278968e-07, 'epoch': 0.45999457553566586, 'num_input_tokens_seen': 5234491392, 'completed': '67.70% (2_496 / 3_687)', 'remaining time': '10:36:53', 'throughput': '8905.68', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 06:35:21,034 >> {'loss': 0.2412, 'grad_norm': 9.705018997192383, 'learning_rate': 3.3297033372694473e-07, 'epoch': 0.4602657987523732, 'num_input_tokens_seen': 5236588544, 'completed': '67.72% (2_497 / 3_687)', 'remaining time': '10:36:21', 'throughput': '8338.83', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 06:35:47,640 >> {'loss': 0.8226, 'grad_norm': 20.149858474731445, 'learning_rate': 3.3261678498320954e-07, 'epoch': 0.46053702196908053, 'num_input_tokens_seen': 5238685696, 'completed': '67.75% (2_498 / 3_687)', 'remaining time': '10:35:45', 'throughput': '9852.88', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 06:36:14,868 >> {'loss': 0.5587, 'grad_norm': 19.860633850097656, 'learning_rate': 3.3226341118121367e-07, 'epoch': 0.4608082451857879, 'num_input_tokens_seen': 5240782848, 'completed': '67.78% (2_499 / 3_687)', 'remaining time': '10:35:09', 'throughput': '9627.96', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 06:36:46,261 >> {'loss': 0.3087, 'grad_norm': 11.387728691101074, 'learning_rate': 3.319102126053389e-07, 'epoch': 0.46107946840249525, 'num_input_tokens_seen': 5242880000, 'completed': '67.81% (2_500 / 3_687)', 'remaining time': '10:34:37', 'throughput': '8350.35', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 06:37:19,114 >> {'loss': 0.2748, 'grad_norm': 16.481552124023438, 'learning_rate': 3.315571895398261e-07, 'epoch': 0.4613506916192026, 'num_input_tokens_seen': 5244977152, 'completed': '67.83% (2_501 / 3_687)', 'remaining time': '10:34:05', 'throughput': '7979.23', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 06:37:50,250 >> {'loss': 0.3922, 'grad_norm': 14.168554306030273, 'learning_rate': 3.312043422687749e-07, 'epoch': 0.46162191483591, 'num_input_tokens_seen': 5247074304, 'completed': '67.86% (2_502 / 3_687)', 'remaining time': '10:33:33', 'throughput': '8419.38', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 06:38:23,262 >> {'loss': 0.2, 'grad_norm': 10.951041221618652, 'learning_rate': 3.3085167107614297e-07, 'epoch': 0.4618931380526173, 'num_input_tokens_seen': 5249171456, 'completed': '67.89% (2_503 / 3_687)', 'remaining time': '10:33:01', 'throughput': '7940.89', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 06:38:52,200 >> {'loss': 0.714, 'grad_norm': 21.78125762939453, 'learning_rate': 3.3049917624574737e-07, 'epoch': 0.46216436126932464, 'num_input_tokens_seen': 5251268608, 'completed': '67.91% (2_504 / 3_687)', 'remaining time': '10:32:27', 'throughput': '9058.79', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 06:39:24,274 >> {'loss': 0.3656, 'grad_norm': 14.395316123962402, 'learning_rate': 3.301468580612619e-07, 'epoch': 0.462435584486032, 'num_input_tokens_seen': 5253365760, 'completed': '67.94% (2_505 / 3_687)', 'remaining time': '10:31:55', 'throughput': '8173.09', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 06:39:55,035 >> {'loss': 0.3252, 'grad_norm': 16.801877975463867, 'learning_rate': 3.2979471680621903e-07, 'epoch': 0.46270680770273936, 'num_input_tokens_seen': 5255462912, 'completed': '67.97% (2_506 / 3_687)', 'remaining time': '10:31:22', 'throughput': '8521.97', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 06:40:23,477 >> {'loss': 0.5567, 'grad_norm': 18.835493087768555, 'learning_rate': 3.2944275276400857e-07, 'epoch': 0.4629780309194467, 'num_input_tokens_seen': 5257560064, 'completed': '68.00% (2_507 / 3_687)', 'remaining time': '10:30:47', 'throughput': '9216.72', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 06:40:53,980 >> {'loss': 0.4733, 'grad_norm': 18.04051971435547, 'learning_rate': 3.290909662178779e-07, 'epoch': 0.46324925413615403, 'num_input_tokens_seen': 5259657216, 'completed': '68.02% (2_508 / 3_687)', 'remaining time': '10:30:14', 'throughput': '8594.09', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 06:41:24,296 >> {'loss': 0.1792, 'grad_norm': 11.844828605651855, 'learning_rate': 3.2873935745093145e-07, 'epoch': 0.4635204773528614, 'num_input_tokens_seen': 5261754368, 'completed': '68.05% (2_509 / 3_687)', 'remaining time': '10:29:41', 'throughput': '8646.94', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 06:41:56,088 >> {'loss': 0.3807, 'grad_norm': 15.879327774047852, 'learning_rate': 3.283879267461305e-07, 'epoch': 0.46379170056956875, 'num_input_tokens_seen': 5263851520, 'completed': '68.08% (2_510 / 3_687)', 'remaining time': '10:29:08', 'throughput': '8245.72', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 06:42:25,974 >> {'loss': 0.2678, 'grad_norm': 14.160933494567871, 'learning_rate': 3.280366743862931e-07, 'epoch': 0.4640629237862761, 'num_input_tokens_seen': 5265948672, 'completed': '68.10% (2_511 / 3_687)', 'remaining time': '10:28:35', 'throughput': '8771.49', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 06:43:00,489 >> {'loss': 0.4197, 'grad_norm': 16.566879272460938, 'learning_rate': 3.276856006540939e-07, 'epoch': 0.4643341470029835, 'num_input_tokens_seen': 5268045824, 'completed': '68.13% (2_512 / 3_687)', 'remaining time': '10:28:05', 'throughput': '7594.93', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 06:43:31,469 >> {'loss': 0.7876, 'grad_norm': 22.101320266723633, 'learning_rate': 3.2733470583206357e-07, 'epoch': 0.4646053702196908, 'num_input_tokens_seen': 5270142976, 'completed': '68.16% (2_513 / 3_687)', 'remaining time': '10:27:32', 'throughput': '8461.96', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 06:44:01,831 >> {'loss': 0.6538, 'grad_norm': 22.703693389892578, 'learning_rate': 3.2698399020258895e-07, 'epoch': 0.46487659343639814, 'num_input_tokens_seen': 5272240128, 'completed': '68.19% (2_514 / 3_687)', 'remaining time': '10:26:58', 'throughput': '8633.87', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 06:44:32,337 >> {'loss': 0.324, 'grad_norm': 14.076583862304688, 'learning_rate': 3.266334540479128e-07, 'epoch': 0.4651478166531055, 'num_input_tokens_seen': 5274337280, 'completed': '68.21% (2_515 / 3_687)', 'remaining time': '10:26:25', 'throughput': '8593.07', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 06:45:02,418 >> {'loss': 0.4449, 'grad_norm': 17.571779251098633, 'learning_rate': 3.262830976501329e-07, 'epoch': 0.46541903986981287, 'num_input_tokens_seen': 5276434432, 'completed': '68.24% (2_516 / 3_687)', 'remaining time': '10:25:52', 'throughput': '8714.67', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 06:45:34,754 >> {'loss': 0.3639, 'grad_norm': 13.718509674072266, 'learning_rate': 3.2593292129120295e-07, 'epoch': 0.46569026308652023, 'num_input_tokens_seen': 5278531584, 'completed': '68.27% (2_517 / 3_687)', 'remaining time': '10:25:20', 'throughput': '8106.95', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 06:46:03,699 >> {'loss': 0.3981, 'grad_norm': 16.070167541503906, 'learning_rate': 3.2558292525293156e-07, 'epoch': 0.46596148630322753, 'num_input_tokens_seen': 5280628736, 'completed': '68.29% (2_518 / 3_687)', 'remaining time': '10:24:46', 'throughput': '9056.49', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 06:46:34,936 >> {'loss': 0.4636, 'grad_norm': 17.910581588745117, 'learning_rate': 3.2523310981698213e-07, 'epoch': 0.4662327095199349, 'num_input_tokens_seen': 5282725888, 'completed': '68.32% (2_519 / 3_687)', 'remaining time': '10:24:13', 'throughput': '8392.15', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 06:47:04,363 >> {'loss': 0.6682, 'grad_norm': 21.05547332763672, 'learning_rate': 3.24883475264873e-07, 'epoch': 0.46650393273664226, 'num_input_tokens_seen': 5284823040, 'completed': '68.35% (2_520 / 3_687)', 'remaining time': '10:23:39', 'throughput': '8908.43', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 06:47:35,403 >> {'loss': 0.4988, 'grad_norm': 14.658893585205078, 'learning_rate': 3.2453402187797684e-07, 'epoch': 0.4667751559533496, 'num_input_tokens_seen': 5286920192, 'completed': '68.38% (2_521 / 3_687)', 'remaining time': '10:23:07', 'throughput': '8445.36', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 06:48:05,633 >> {'loss': 0.4897, 'grad_norm': 17.221160888671875, 'learning_rate': 3.241847499375201e-07, 'epoch': 0.467046379170057, 'num_input_tokens_seen': 5289017344, 'completed': '68.40% (2_522 / 3_687)', 'remaining time': '10:22:33', 'throughput': '8671.55', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 06:48:34,553 >> {'loss': 0.2647, 'grad_norm': 14.185545921325684, 'learning_rate': 3.238356597245837e-07, 'epoch': 0.4673176023867643, 'num_input_tokens_seen': 5291114496, 'completed': '68.43% (2_523 / 3_687)', 'remaining time': '10:21:59', 'throughput': '9064.40', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 06:49:03,957 >> {'loss': 0.4688, 'grad_norm': 17.018024444580078, 'learning_rate': 3.2348675152010217e-07, 'epoch': 0.46758882560347165, 'num_input_tokens_seen': 5293211648, 'completed': '68.46% (2_524 / 3_687)', 'remaining time': '10:21:25', 'throughput': '8915.27', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 06:49:35,354 >> {'loss': 0.5423, 'grad_norm': 21.121870040893555, 'learning_rate': 3.2313802560486353e-07, 'epoch': 0.467860048820179, 'num_input_tokens_seen': 5295308800, 'completed': '68.48% (2_525 / 3_687)', 'remaining time': '10:20:53', 'throughput': '8349.22', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 06:50:06,196 >> {'loss': 0.6087, 'grad_norm': 41.621604919433594, 'learning_rate': 3.2278948225950916e-07, 'epoch': 0.46813127203688637, 'num_input_tokens_seen': 5297405952, 'completed': '68.51% (2_526 / 3_687)', 'remaining time': '10:20:20', 'throughput': '8499.76', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 06:50:36,336 >> {'loss': 0.4773, 'grad_norm': 15.993080139160156, 'learning_rate': 3.2244112176453343e-07, 'epoch': 0.46840249525359373, 'num_input_tokens_seen': 5299503104, 'completed': '68.54% (2_527 / 3_687)', 'remaining time': '10:19:46', 'throughput': '8697.53', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 06:51:06,493 >> {'loss': 0.6761, 'grad_norm': 19.632001876831055, 'learning_rate': 3.2209294440028366e-07, 'epoch': 0.46867371847030104, 'num_input_tokens_seen': 5301600256, 'completed': '68.57% (2_528 / 3_687)', 'remaining time': '10:19:13', 'throughput': '8692.65', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 06:51:39,333 >> {'loss': 0.3395, 'grad_norm': 14.451026916503906, 'learning_rate': 3.2174495044695973e-07, 'epoch': 0.4689449416870084, 'num_input_tokens_seen': 5303697408, 'completed': '68.59% (2_529 / 3_687)', 'remaining time': '10:18:42', 'throughput': '7982.37', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 06:52:10,971 >> {'loss': 0.483, 'grad_norm': 18.51044464111328, 'learning_rate': 3.2139714018461396e-07, 'epoch': 0.46921616490371576, 'num_input_tokens_seen': 5305794560, 'completed': '68.62% (2_530 / 3_687)', 'remaining time': '10:18:09', 'throughput': '8286.31', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 06:52:42,584 >> {'loss': 0.5314, 'grad_norm': 16.25364875793457, 'learning_rate': 3.2104951389315073e-07, 'epoch': 0.4694873881204231, 'num_input_tokens_seen': 5307891712, 'completed': '68.65% (2_531 / 3_687)', 'remaining time': '10:17:37', 'throughput': '8292.15', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 06:53:10,760 >> {'loss': 0.603, 'grad_norm': 21.938037872314453, 'learning_rate': 3.207020718523266e-07, 'epoch': 0.4697586113371305, 'num_input_tokens_seen': 5309988864, 'completed': '68.67% (2_532 / 3_687)', 'remaining time': '10:17:02', 'throughput': '9303.42', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 06:53:42,394 >> {'loss': 0.3931, 'grad_norm': 15.501047134399414, 'learning_rate': 3.2035481434174966e-07, 'epoch': 0.4700298345538378, 'num_input_tokens_seen': 5312086016, 'completed': '68.70% (2_533 / 3_687)', 'remaining time': '10:16:30', 'throughput': '8286.76', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 06:54:12,167 >> {'loss': 0.4532, 'grad_norm': 16.000394821166992, 'learning_rate': 3.200077416408794e-07, 'epoch': 0.47030105777054515, 'num_input_tokens_seen': 5314183168, 'completed': '68.73% (2_534 / 3_687)', 'remaining time': '10:15:56', 'throughput': '8804.71', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 06:54:41,966 >> {'loss': 0.631, 'grad_norm': 21.34868049621582, 'learning_rate': 3.196608540290266e-07, 'epoch': 0.4705722809872525, 'num_input_tokens_seen': 5316280320, 'completed': '68.76% (2_535 / 3_687)', 'remaining time': '10:15:23', 'throughput': '8798.00', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 06:55:14,571 >> {'loss': 0.5566, 'grad_norm': 23.8501033782959, 'learning_rate': 3.193141517853536e-07, 'epoch': 0.4708435042039599, 'num_input_tokens_seen': 5318377472, 'completed': '68.78% (2_536 / 3_687)', 'remaining time': '10:14:51', 'throughput': '8039.20', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 06:55:45,261 >> {'loss': 0.4631, 'grad_norm': 14.53099250793457, 'learning_rate': 3.1896763518887305e-07, 'epoch': 0.47111472742066723, 'num_input_tokens_seen': 5320474624, 'completed': '68.81% (2_537 / 3_687)', 'remaining time': '10:14:18', 'throughput': '8541.73', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 06:56:12,292 >> {'loss': 0.9052, 'grad_norm': 25.955514907836914, 'learning_rate': 3.186213045184484e-07, 'epoch': 0.47138595063737454, 'num_input_tokens_seen': 5322571776, 'completed': '68.84% (2_538 / 3_687)', 'remaining time': '10:13:43', 'throughput': '9697.83', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 06:56:45,509 >> {'loss': 0.329, 'grad_norm': 12.901291847229004, 'learning_rate': 3.1827516005279306e-07, 'epoch': 0.4716571738540819, 'num_input_tokens_seen': 5324668928, 'completed': '68.86% (2_539 / 3_687)', 'remaining time': '10:13:12', 'throughput': '7891.78', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 06:57:16,267 >> {'loss': 0.3753, 'grad_norm': 14.975889205932617, 'learning_rate': 3.1792920207047114e-07, 'epoch': 0.47192839707078926, 'num_input_tokens_seen': 5326766080, 'completed': '68.89% (2_540 / 3_687)', 'remaining time': '10:12:39', 'throughput': '8522.91', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 06:57:48,738 >> {'loss': 0.4148, 'grad_norm': 17.118913650512695, 'learning_rate': 3.175834308498964e-07, 'epoch': 0.4721996202874966, 'num_input_tokens_seen': 5328863232, 'completed': '68.92% (2_541 / 3_687)', 'remaining time': '10:12:07', 'throughput': '8073.19', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 06:58:18,778 >> {'loss': 0.3607, 'grad_norm': 16.228837966918945, 'learning_rate': 3.172378466693325e-07, 'epoch': 0.472470843504204, 'num_input_tokens_seen': 5330960384, 'completed': '68.94% (2_542 / 3_687)', 'remaining time': '10:11:33', 'throughput': '8726.50', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 06:58:46,741 >> {'loss': 0.9653, 'grad_norm': 22.712890625, 'learning_rate': 3.168924498068923e-07, 'epoch': 0.4727420667209113, 'num_input_tokens_seen': 5333057536, 'completed': '68.97% (2_543 / 3_687)', 'remaining time': '10:10:59', 'throughput': '9374.65', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 06:59:17,948 >> {'loss': 0.4317, 'grad_norm': 18.086130142211914, 'learning_rate': 3.1654724054053805e-07, 'epoch': 0.47301328993761865, 'num_input_tokens_seen': 5335154688, 'completed': '69.00% (2_544 / 3_687)', 'remaining time': '10:10:26', 'throughput': '8400.13', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2024-12-26 06:59:50,306 >> {'loss': 0.4091, 'grad_norm': 16.25860595703125, 'learning_rate': 3.1620221914808115e-07, 'epoch': 0.473284513154326, 'num_input_tokens_seen': 5337251840, 'completed': '69.03% (2_545 / 3_687)', 'remaining time': '10:09:54', 'throughput': '8101.33', 'gpu_mem_free': '5581MB'}