diff --git "a/wandb/run-20241225_152744-sac86tlv/logs/debug-internal.log" "b/wandb/run-20241225_152744-sac86tlv/logs/debug-internal.log" new file mode 100644--- /dev/null +++ "b/wandb/run-20241225_152744-sac86tlv/logs/debug-internal.log" @@ -0,0 +1,46989 @@ +2024-12-25 15:27:44,404 INFO StreamThr :316822 [internal.py:wandb_internal():85] W&B internal server running at pid: 316822, started at: 2024-12-25 15:27:44.403583 +2024-12-25 15:27:44,407 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status +2024-12-25 15:27:44,414 INFO WriterThread:316822 [datastore.py:open_for_write():87] open: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/run-sac86tlv.wandb +2024-12-25 15:27:44,415 DEBUG SenderThread:316822 [sender.py:send():379] send: header +2024-12-25 15:27:44,443 DEBUG SenderThread:316822 [sender.py:send():379] send: run +2024-12-25 15:27:44,680 INFO SenderThread:316822 [dir_watcher.py:__init__():211] watching files in: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files +2024-12-25 15:27:44,680 INFO SenderThread:316822 [sender.py:_start_run_threads():1188] run started: sac86tlv with start time 1735140464.405984 +2024-12-25 15:27:44,703 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: check_version +2024-12-25 15:27:44,704 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: check_version +2024-12-25 15:27:44,780 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: run_start +2024-12-25 15:27:44,813 DEBUG HandlerThread:316822 [system_info.py:__init__():26] System info init +2024-12-25 15:27:44,813 DEBUG HandlerThread:316822 [system_info.py:__init__():41] System info init done +2024-12-25 15:27:44,813 INFO HandlerThread:316822 [system_monitor.py:start():194] Starting system monitor +2024-12-25 15:27:44,813 INFO SystemMonitor:316822 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-12-25 15:27:44,814 INFO HandlerThread:316822 [system_monitor.py:probe():214] Collecting system info +2024-12-25 15:27:44,815 INFO SystemMonitor:316822 [interfaces.py:start():188] Started cpu monitoring +2024-12-25 15:27:44,815 INFO SystemMonitor:316822 [interfaces.py:start():188] Started disk monitoring +2024-12-25 15:27:44,816 INFO SystemMonitor:316822 [interfaces.py:start():188] Started gpu monitoring +2024-12-25 15:27:44,820 INFO SystemMonitor:316822 [interfaces.py:start():188] Started memory monitoring +2024-12-25 15:27:44,821 INFO SystemMonitor:316822 [interfaces.py:start():188] Started network monitoring +2024-12-25 15:27:44,870 DEBUG HandlerThread:316822 [system_info.py:probe():152] Probing system +2024-12-25 15:27:44,873 DEBUG HandlerThread:316822 [system_info.py:_probe_git():137] Probing git +2024-12-25 15:27:44,890 DEBUG HandlerThread:316822 [system_info.py:_probe_git():145] Probing git done +2024-12-25 15:27:44,890 DEBUG HandlerThread:316822 [system_info.py:probe():200] Probing system done +2024-12-25 15:27:44,890 DEBUG HandlerThread:316822 [system_monitor.py:probe():223] {'os': 'Linux-6.8.0-48-generic-x86_64-with-glibc2.39', 'python': '3.10.0', 'heartbeatAt': '2024-12-25T15:27:44.870156', 'startedAt': '2024-12-25T15:27:44.387439', 'docker': None, 'cuda': None, 'args': ('--model_family', 'llama', '--apply_instruct_masks', '--token_scaled_loss', '--seq_parallel_size', '8', '--report_to', 'wandb', '--do_train', '--model_name_or_path', '/scratch3/workspace/ctpham_umass_edu-ft/models/Llama-3-8B-ProLong-512k-Base/', '--config_name', '/scratch3/workspace/ctpham_umass_edu-ft/models/Llama-3-8B-ProLong-512k-Base/', '--tokenizer_name', '/scratch3/workspace/ctpham_umass_edu-ft/models/Llama-3-8B-ProLong-512k-Base/', '--run_name', 'balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_', '--output_dir', '/scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_', '--config_overrides_json', '', '--gradient_accumulation_steps', '2', '--per_device_train_batch_size', '1', '--bf16', '--learning_rate', '1e-6', '--min_lr_ratio', '0.1', '--lr_scheduler_type', 'cosine', '--max_grad_norm', '1.0', '--adam_beta1', '0.9', '--adam_beta2', '0.95', '--weight_decay', '0.1', '--warmup_ratio', '0.05', '--optim', 'adamw_torch', '--logging_steps', '1', '--log_level', 'info', '--save_steps', '200', '--dataloader_num_workers', '1', '--disable_tqdm', 'true', '--use_fast_tokenizer', 'false', '--remove_unused_columns', 'false', '--ddp_find_unused_parameters', 'false', '--fsdp', 'auto_wrap offload', '--gradient_checkpointing', '--tokenized_mds_train', '/work/pi_miyyer_umass_edu/ctpham/BookClaim-dev/data/ft/bookclaim_balanced_pack_complete', '--cuda_empty_cache', '--num_train_epochs', '1'), 'state': 'running', 'program': '/work/pi_miyyer_umass_edu/ctpham/BookClaim-dev/prolong-final/finetune.py', 'codePathLocal': 'finetune.py', 'codePath': 'prolong-final/finetune.py', 'git': {'remote': 'https://github.com/chtmp223/BookGen-dev.git', 'commit': '438e68ae65a707227d28baa427c5f6fdb0932c7b'}, 'email': 'ctpham@umass.edu', 'root': '/work/pi_miyyer_umass_edu/ctpham/BookClaim-dev', 'host': 'gpu020', 'username': 'ctpham_umass_edu', 'executable': '/scratch3/workspace/ctpham_umass_edu-ft/envs/prolong-final/bin/python3.10', 'cpu_count': 112, 'cpu_count_logical': 112, 'cpu_freq': {'current': 997.5532857142858, 'min': 800.0, 'max': 3800.0}, 'cpu_freq_per_core': [{'current': 765.075, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 783.551, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 791.32, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 791.402, 'min': 800.0, 'max': 3800.0}, {'current': 788.612, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 783.817, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 1700.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 783.39, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 783.853, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 1600.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 1995.789, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 2400.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 2400.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 2400.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 2400.0, 'min': 800.0, 'max': 3800.0}, {'current': 3800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 784.222, 'min': 800.0, 'max': 3800.0}, {'current': 2400.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 2054.42, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 1998.667, 'min': 800.0, 'max': 3800.0}, {'current': 3800.0, 'min': 800.0, 'max': 3800.0}, {'current': 2009.408, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 1991.42, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 783.068, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 783.462, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}], 'disk': {'/': {'total': 438.487850189209, 'used': 18.09857940673828}}, 'gpu': 'NVIDIA A100-SXM4-80GB', 'gpu_count': 8, 'gpu_devices': [{'name': 'NVIDIA A100-SXM4-80GB', 'memory_total': 85899345920}, {'name': 'NVIDIA A100-SXM4-80GB', 'memory_total': 85899345920}, {'name': 'NVIDIA A100-SXM4-80GB', 'memory_total': 85899345920}, {'name': 'NVIDIA A100-SXM4-80GB', 'memory_total': 85899345920}, {'name': 'NVIDIA A100-SXM4-80GB', 'memory_total': 85899345920}, {'name': 'NVIDIA A100-SXM4-80GB', 'memory_total': 85899345920}, {'name': 'NVIDIA A100-SXM4-80GB', 'memory_total': 85899345920}, {'name': 'NVIDIA A100-SXM4-80GB', 'memory_total': 85899345920}], 'memory': {'total': 2015.3287239074707}} +2024-12-25 15:27:44,890 INFO HandlerThread:316822 [system_monitor.py:probe():224] Finished collecting system info +2024-12-25 15:27:44,890 INFO HandlerThread:316822 [system_monitor.py:probe():227] Publishing system info +2024-12-25 15:27:44,891 DEBUG HandlerThread:316822 [system_info.py:_save_conda():209] Saving list of conda packages installed into the current environment +2024-12-25 15:27:45,682 INFO Thread-12 :316822 [dir_watcher.py:_on_file_created():271] file/dir created: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/conda-environment.yaml +2024-12-25 15:27:51,919 DEBUG HandlerThread:316822 [system_info.py:_save_conda():224] Saving conda packages done +2024-12-25 15:27:51,922 INFO HandlerThread:316822 [system_monitor.py:probe():229] Finished publishing system info +2024-12-25 15:27:51,928 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:27:51,928 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 15:27:51,929 DEBUG SenderThread:316822 [sender.py:send():379] send: files +2024-12-25 15:27:51,929 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2024-12-25 15:27:52,192 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: python_packages +2024-12-25 15:27:52,193 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: python_packages +2024-12-25 15:27:52,194 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 15:27:52,194 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:27:52,195 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 15:27:52,251 INFO wandb-upload_0:316822 [upload_job.py:push():130] Uploaded file /tmp/tmpxw4q47rvwandb/7s5h95lp-wandb-metadata.json +2024-12-25 15:27:52,359 DEBUG SenderThread:316822 [sender.py:send():379] send: telemetry +2024-12-25 15:27:52,359 DEBUG SenderThread:316822 [sender.py:send():379] send: config +2024-12-25 15:27:52,361 DEBUG SenderThread:316822 [sender.py:send():379] send: telemetry +2024-12-25 15:27:52,361 DEBUG SenderThread:316822 [sender.py:send():379] send: metric +2024-12-25 15:27:52,361 DEBUG SenderThread:316822 [sender.py:send():379] send: telemetry +2024-12-25 15:27:52,362 DEBUG SenderThread:316822 [sender.py:send():379] send: metric +2024-12-25 15:27:52,362 WARNING SenderThread:316822 [sender.py:send_metric():1405] Seen metric with glob (shouldn't happen) +2024-12-25 15:27:52,362 DEBUG SenderThread:316822 [sender.py:send():379] send: telemetry +2024-12-25 15:27:52,362 DEBUG SenderThread:316822 [sender.py:send():379] send: telemetry +2024-12-25 15:27:52,362 DEBUG SenderThread:316822 [sender.py:send():379] send: config +2024-12-25 15:27:52,684 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/conda-environment.yaml +2024-12-25 15:27:52,684 INFO Thread-12 :316822 [dir_watcher.py:_on_file_created():271] file/dir created: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-metadata.json +2024-12-25 15:27:52,685 INFO Thread-12 :316822 [dir_watcher.py:_on_file_created():271] file/dir created: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/requirements.txt +2024-12-25 15:27:52,685 INFO Thread-12 :316822 [dir_watcher.py:_on_file_created():271] file/dir created: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 15:27:53,194 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:27:54,194 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:27:54,686 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 15:27:55,194 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:27:55,364 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:27:56,194 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:27:57,194 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:27:58,194 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:27:59,194 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:28:00,194 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:28:01,194 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:28:01,195 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:28:02,195 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:28:03,195 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:28:04,195 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:28:05,195 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:28:06,195 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:28:06,600 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:28:06,693 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 15:28:07,193 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 15:28:07,194 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 15:28:07,234 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:28:08,195 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:28:09,195 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:28:10,195 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:28:11,195 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:28:12,194 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:28:12,195 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:28:13,195 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:28:14,195 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:28:15,196 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:28:16,197 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:28:17,200 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:28:17,208 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:28:17,699 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/config.yaml +2024-12-25 15:28:18,196 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:28:19,196 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:28:20,196 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:28:21,196 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:28:22,193 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 15:28:22,194 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 15:28:22,233 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:28:23,196 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:28:23,324 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:28:24,197 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:28:25,197 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:28:26,197 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:28:27,197 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:28:28,197 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:28:29,198 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:28:29,199 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:28:30,197 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:28:31,197 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:28:32,197 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:28:33,197 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:28:34,198 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:28:35,197 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:28:35,198 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:28:35,986 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 15:28:35,988 DEBUG SenderThread:316822 [sender.py:send():379] send: metric +2024-12-25 15:28:35,989 DEBUG SenderThread:316822 [sender.py:send():379] send: metric +2024-12-25 15:28:35,989 DEBUG SenderThread:316822 [sender.py:send():379] send: metric +2024-12-25 15:28:35,989 DEBUG SenderThread:316822 [sender.py:send():379] send: metric +2024-12-25 15:28:35,989 DEBUG SenderThread:316822 [sender.py:send():379] send: metric +2024-12-25 15:28:35,989 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 15:28:35,990 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 15:28:35,995 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 15:28:36,386 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:28:36,729 INFO Thread-12 :316822 [dir_watcher.py:_on_file_created():271] file/dir created: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 15:28:37,193 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 15:28:37,194 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 15:28:37,386 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:28:38,386 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:28:38,731 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 15:28:39,386 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:28:40,273 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:28:40,386 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:28:41,386 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:28:42,386 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:28:43,386 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:28:44,386 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:28:44,821 DEBUG SystemMonitor:316822 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-12-25 15:28:44,824 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 15:28:45,387 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:28:45,826 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:28:46,387 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:28:47,387 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:28:48,387 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:28:49,387 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:28:50,387 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:28:51,204 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:28:51,387 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:28:51,742 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/config.yaml +2024-12-25 15:28:52,193 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 15:28:52,194 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 15:28:52,387 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:28:53,387 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:28:54,387 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:28:55,388 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:28:56,355 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:28:56,388 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:28:57,388 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:28:58,388 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:28:59,388 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:29:00,388 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:29:01,395 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:29:02,201 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:29:02,394 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:29:03,394 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:29:04,394 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:29:05,398 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:29:06,398 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:29:06,899 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 15:29:06,899 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 15:29:06,900 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 15:29:06,903 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 15:29:07,312 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:29:07,314 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 15:29:07,314 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 15:29:07,398 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:29:07,761 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 15:29:08,398 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:29:08,762 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 15:29:09,398 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:29:10,398 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:29:11,398 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:29:12,398 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:29:13,310 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:29:13,398 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:29:14,399 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:29:14,826 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 15:29:15,399 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:29:16,399 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:29:17,399 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:29:18,312 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:29:18,400 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:29:19,632 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:29:20,399 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:29:21,399 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:29:22,309 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 15:29:22,310 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 15:29:22,400 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:29:23,400 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:29:23,445 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:29:24,400 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:29:25,400 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:29:26,400 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:29:27,400 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:29:28,400 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:29:29,311 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:29:29,400 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:29:30,400 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:29:31,401 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:29:32,404 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:29:33,405 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:29:34,318 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:29:34,411 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:29:35,410 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:29:36,409 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:29:37,309 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 15:29:37,310 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 15:29:37,409 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:29:37,539 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 15:29:37,541 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 15:29:37,542 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 15:29:37,544 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 15:29:37,787 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 15:29:38,409 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:29:38,788 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 15:29:39,409 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:29:39,581 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:29:40,409 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:29:41,409 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:29:42,409 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:29:43,410 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:29:44,410 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:29:44,829 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 15:29:44,832 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:29:45,410 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:29:46,410 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:29:47,410 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:29:48,410 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:29:49,410 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:29:50,318 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:29:50,410 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:29:51,410 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:29:52,310 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 15:29:52,310 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 15:29:52,410 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:29:53,410 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:29:54,411 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:29:55,411 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:29:55,432 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:29:56,411 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:29:57,411 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:29:58,411 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:29:59,418 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:30:00,415 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:30:01,327 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:30:01,414 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:30:02,419 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:30:03,421 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:30:04,419 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:30:04,732 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 15:30:04,734 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 15:30:04,734 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 15:30:04,735 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 15:30:04,817 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 15:30:05,419 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:30:06,419 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:30:06,773 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:30:06,818 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 15:30:07,310 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 15:30:07,311 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 15:30:07,419 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:30:08,419 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:30:09,419 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:30:10,419 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:30:11,420 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:30:12,323 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:30:12,420 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:30:13,420 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:30:14,420 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:30:14,834 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 15:30:15,420 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:30:16,420 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:30:17,420 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:30:18,323 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:30:18,420 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:30:19,421 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:30:20,421 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:30:21,421 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:30:22,310 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 15:30:22,311 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 15:30:22,420 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:30:23,416 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:30:23,421 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:30:24,421 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:30:25,421 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:30:26,421 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:30:27,421 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:30:28,421 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:30:29,323 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:30:29,421 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:30:30,421 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:30:31,433 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:30:32,426 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:30:33,430 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:30:34,329 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:30:34,437 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:30:35,431 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:30:36,431 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:30:36,798 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 15:30:36,799 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 15:30:36,800 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 15:30:36,805 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 15:30:36,846 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 15:30:37,310 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 15:30:37,311 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 15:30:37,431 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:30:38,431 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:30:38,848 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 15:30:39,383 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:30:39,431 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:30:40,431 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:30:41,431 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:30:42,432 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:30:43,432 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:30:44,432 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:30:44,837 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 15:30:44,840 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:30:45,432 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:30:46,432 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:30:47,432 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:30:48,433 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:30:49,432 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:30:50,326 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:30:50,432 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:30:51,433 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:30:52,310 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 15:30:52,311 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 15:30:52,433 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:30:53,433 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:30:54,433 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:30:55,433 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:30:55,441 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:30:56,433 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:30:57,433 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:30:58,433 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:30:59,434 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:31:00,437 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:31:01,331 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:31:01,438 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:31:02,437 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:31:03,439 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:31:04,439 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:31:05,440 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:31:05,593 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 15:31:05,594 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 15:31:05,595 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 15:31:05,597 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 15:31:05,872 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 15:31:06,440 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:31:06,635 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:31:06,873 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 15:31:07,310 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 15:31:07,311 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 15:31:07,439 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:31:08,439 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:31:09,440 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:31:10,440 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:31:11,440 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:31:12,331 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:31:12,440 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:31:13,440 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:31:14,440 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:31:14,841 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 15:31:15,440 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:31:16,440 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:31:17,440 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:31:18,331 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:31:18,440 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:31:19,440 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:31:20,503 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:31:21,502 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:31:22,310 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 15:31:22,311 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 15:31:22,502 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:31:23,476 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:31:23,502 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:31:24,502 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:31:25,502 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:31:26,503 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:31:27,503 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:31:28,503 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:31:29,334 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:31:29,503 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:31:30,503 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:31:31,503 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:31:32,508 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:31:33,506 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:31:34,351 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:31:34,508 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:31:35,509 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:31:36,506 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:31:37,311 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 15:31:37,311 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 15:31:37,506 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:31:37,803 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 15:31:37,804 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 15:31:37,805 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 15:31:37,807 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 15:31:37,906 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 15:31:38,507 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:31:38,907 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 15:31:39,507 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:31:39,845 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:31:40,507 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:31:41,507 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:31:42,507 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:31:43,507 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:31:44,507 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:31:44,845 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 15:31:44,848 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:31:45,507 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:31:46,507 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:31:47,507 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:31:48,507 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:31:49,507 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:31:50,338 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:31:50,508 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:31:51,508 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:31:52,392 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 15:31:52,393 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 15:31:52,508 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:31:53,508 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:31:54,508 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:31:55,472 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:31:55,508 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:31:56,508 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:31:57,508 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:31:58,508 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:31:59,508 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:32:00,508 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:32:01,391 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:32:01,509 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:32:02,509 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:32:03,509 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:32:04,509 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:32:05,516 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:32:06,393 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:32:06,520 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:32:07,401 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 15:32:07,402 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 15:32:07,515 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:32:08,517 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:32:09,518 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:32:10,515 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:32:10,895 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 15:32:10,896 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 15:32:10,897 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 15:32:10,902 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 15:32:10,938 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 15:32:11,854 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:32:11,937 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:32:12,857 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:32:12,939 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 15:32:13,857 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:32:14,849 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 15:32:14,857 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:32:15,857 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:32:16,857 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:32:17,422 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:32:17,857 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:32:18,857 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:32:19,857 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:32:20,857 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:32:21,858 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:32:22,395 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 15:32:22,395 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 15:32:22,500 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:32:22,858 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:32:23,858 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:32:24,858 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:32:25,858 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:32:26,858 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:32:27,858 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:32:28,422 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:32:28,858 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:32:29,859 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:32:30,859 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:32:31,859 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:32:32,859 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:32:33,859 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:32:34,422 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:32:34,859 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:32:35,859 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:32:36,859 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:32:37,395 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 15:32:37,395 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 15:32:37,859 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:32:38,859 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:32:39,558 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:32:39,860 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:32:40,861 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:32:41,865 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:32:42,865 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:32:43,873 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:32:44,854 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 15:32:44,855 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:32:44,865 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:32:45,863 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 15:32:45,864 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 15:32:45,864 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 15:32:45,865 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 15:32:45,904 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:32:45,962 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 15:32:46,867 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:32:46,963 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 15:32:47,870 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:32:48,870 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:32:49,870 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:32:50,431 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:32:50,870 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:32:51,870 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:32:52,395 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 15:32:52,396 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 15:32:52,870 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:32:53,870 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:32:54,870 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:32:55,464 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:32:55,870 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:32:56,871 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:32:57,871 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:32:59,045 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:32:59,954 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:33:00,954 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:33:01,430 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:33:01,955 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:33:02,954 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:33:03,954 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:33:04,955 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:33:05,955 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:33:06,430 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:33:06,955 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:33:07,395 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 15:33:07,396 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 15:33:07,955 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:33:08,955 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:33:09,955 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:33:10,961 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:33:11,504 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:33:11,961 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:33:12,961 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:33:13,961 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:33:14,864 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 15:33:14,961 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:33:15,961 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:33:16,398 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 15:33:16,399 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 15:33:16,399 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 15:33:16,401 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 15:33:16,961 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:33:16,985 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 15:33:16,986 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 15:33:17,440 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:33:17,967 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:33:18,966 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:33:19,966 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:33:20,966 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:33:21,966 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:33:22,395 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 15:33:22,396 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 15:33:22,545 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:33:22,966 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:33:23,967 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:33:24,967 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:33:25,967 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:33:26,967 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:33:27,967 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:33:28,444 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:33:28,967 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:33:29,968 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:33:30,967 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:33:31,967 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:33:32,967 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:33:33,444 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:33:33,967 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:33:34,968 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:33:35,968 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:33:36,968 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:33:37,395 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 15:33:37,396 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 15:33:37,968 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:33:38,485 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:33:38,968 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:33:39,968 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:33:40,968 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:33:41,968 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:33:42,974 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:33:43,978 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:33:44,464 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:33:44,866 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 15:33:44,976 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:33:45,977 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:33:46,977 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:33:47,977 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:33:47,984 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 15:33:47,985 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 15:33:47,985 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 15:33:47,986 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 15:33:48,014 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 15:33:48,977 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:33:49,016 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 15:33:49,977 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:33:50,026 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:33:50,977 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:33:51,977 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:33:52,396 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 15:33:52,396 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 15:33:52,978 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:33:53,978 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:33:54,978 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:33:55,522 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:33:55,978 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:33:56,978 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:33:57,978 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:33:58,978 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:33:59,978 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:34:00,978 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:34:01,450 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:34:01,978 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:34:02,978 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:34:03,978 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:34:04,979 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:34:05,979 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:34:06,979 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:34:07,396 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 15:34:07,396 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 15:34:07,546 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:34:07,979 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:34:08,979 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:34:09,979 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:34:10,979 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:34:11,979 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:34:12,979 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:34:13,450 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:34:13,994 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:34:14,867 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 15:34:14,985 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:34:15,985 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:34:16,985 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:34:17,991 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:34:18,458 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:34:18,989 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:34:19,597 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 15:34:19,599 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 15:34:19,599 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 15:34:19,601 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 15:34:20,008 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:34:20,037 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 15:34:21,011 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:34:21,038 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 15:34:22,011 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:34:22,396 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 15:34:22,396 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 15:34:23,011 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:34:23,518 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:34:24,011 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:34:25,011 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:34:26,011 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:34:27,011 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:34:28,011 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:34:29,011 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:34:29,455 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:34:30,011 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:34:31,012 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:34:32,012 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:34:33,012 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:34:34,012 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:34:35,012 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:34:35,455 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:34:36,012 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:34:37,012 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:34:37,396 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 15:34:37,397 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 15:34:38,012 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:34:39,012 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:34:40,012 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:34:40,511 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:34:41,012 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:34:42,013 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:34:43,013 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:34:44,013 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:34:44,869 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 15:34:45,024 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:34:45,872 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:34:46,015 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:34:47,015 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:34:48,015 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:34:49,015 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:34:50,015 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:34:50,554 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 15:34:50,555 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 15:34:50,556 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 15:34:50,559 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 15:34:51,028 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:34:51,064 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 15:34:51,065 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 15:34:51,596 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:34:52,027 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:34:52,396 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 15:34:52,397 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 15:34:53,027 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:34:54,027 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:34:55,027 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:34:56,027 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:34:57,027 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:34:57,459 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:34:58,027 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:34:59,027 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:35:00,028 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:35:01,028 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:35:02,028 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:35:03,028 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:35:03,459 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:35:04,028 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:35:05,028 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:35:06,028 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:35:07,028 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:35:07,396 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 15:35:07,397 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 15:35:08,029 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:35:08,495 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:35:09,029 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:35:10,029 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:35:11,029 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:35:12,029 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:35:13,029 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:35:14,029 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:35:14,459 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:35:14,873 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 15:35:15,029 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:35:16,038 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:35:17,040 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:35:18,036 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:35:19,036 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:35:19,474 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:35:20,039 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:35:21,037 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:35:21,194 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 15:35:21,195 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 15:35:21,196 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 15:35:21,201 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 15:35:22,037 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:35:22,090 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 15:35:22,396 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 15:35:22,397 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 15:35:23,039 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:35:24,039 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:35:24,091 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 15:35:24,559 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:35:25,039 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:35:26,039 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:35:27,039 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:35:28,039 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:35:29,039 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:35:30,039 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:35:30,461 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:35:31,039 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:35:32,040 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:35:33,039 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:35:34,040 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:35:35,040 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:35:36,040 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:35:36,461 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:35:37,040 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:35:37,396 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 15:35:37,397 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 15:35:38,040 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:35:39,040 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:35:40,040 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:35:41,040 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:35:41,556 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:35:42,040 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:35:43,040 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:35:44,041 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:35:44,879 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 15:35:45,041 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:35:46,044 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:35:46,886 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:35:47,046 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:35:48,045 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:35:49,049 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:35:50,049 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:35:50,992 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 15:35:50,993 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 15:35:50,993 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 15:35:50,995 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 15:35:51,112 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 15:35:51,394 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:35:52,033 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:35:52,393 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:35:52,397 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 15:35:52,397 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 15:35:53,393 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:35:54,114 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 15:35:54,393 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:35:55,393 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:35:56,393 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:35:57,394 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:35:57,463 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:35:58,394 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:35:59,394 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:36:00,394 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:36:01,394 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:36:02,394 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:36:03,394 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:36:03,463 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:36:04,394 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:36:05,395 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:36:06,395 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:36:07,395 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:36:07,397 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 15:36:07,397 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 15:36:08,395 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:36:08,568 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:36:09,395 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:36:10,395 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:36:11,395 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:36:12,395 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:36:13,395 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:36:14,395 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:36:14,463 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:36:14,881 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 15:36:15,395 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:36:16,395 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:36:17,395 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:36:18,403 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:36:19,399 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:36:19,467 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:36:20,402 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:36:21,402 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:36:22,402 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 15:36:22,410 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 15:36:22,443 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:36:23,402 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:36:24,017 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 15:36:24,019 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 15:36:24,019 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 15:36:24,025 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 15:36:24,140 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 15:36:24,416 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:36:25,059 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:36:25,415 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:36:26,142 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 15:36:26,416 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:36:27,416 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:36:28,416 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:36:29,416 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:36:30,416 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:36:30,465 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:36:31,416 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:36:32,416 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:36:33,416 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:36:34,416 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:36:35,416 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:36:36,416 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:36:36,465 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:36:37,400 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 15:36:37,400 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 15:36:37,440 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:36:38,417 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:36:39,417 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:36:40,417 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:36:41,417 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:36:41,529 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:36:42,417 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:36:43,417 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:36:44,417 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:36:44,885 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 15:36:45,417 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:36:46,417 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:36:46,887 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:36:47,417 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:36:48,417 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:36:49,418 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:36:50,418 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:36:51,423 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:36:52,410 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 15:36:52,411 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 15:36:52,446 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:36:52,555 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:36:53,426 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:36:54,426 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:36:55,424 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:36:56,424 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:36:56,450 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 15:36:56,452 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 15:36:56,453 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 15:36:56,458 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 15:36:57,165 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 15:36:57,428 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:36:58,166 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 15:36:58,427 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:36:58,465 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:36:59,427 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:37:00,427 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:37:01,427 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:37:02,427 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:37:03,428 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:37:04,428 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:37:04,465 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:37:05,428 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:37:06,428 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:37:07,404 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 15:37:07,404 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 15:37:07,444 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:37:08,428 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:37:09,428 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:37:09,514 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:37:10,428 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:37:11,428 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:37:12,428 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:37:13,429 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:37:14,429 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:37:14,889 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 15:37:14,891 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:37:15,429 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:37:16,429 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:37:17,429 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:37:18,442 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:37:19,434 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:37:20,436 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:37:20,471 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:37:21,436 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:37:22,404 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 15:37:22,404 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 15:37:22,444 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:37:23,348 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 15:37:23,350 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 15:37:23,350 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 15:37:23,352 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 15:37:23,746 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:37:24,197 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 15:37:24,494 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:37:25,494 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:37:26,198 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 15:37:26,390 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:37:26,494 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:37:27,494 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:37:28,494 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:37:29,494 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:37:30,494 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:37:31,469 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:37:31,494 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:37:32,495 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:37:33,495 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:37:34,495 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:37:35,496 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:37:36,495 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:37:37,404 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 15:37:37,405 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 15:37:37,495 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:37:37,505 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:37:38,495 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:37:39,495 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:37:40,495 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:37:41,496 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:37:42,496 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:37:43,469 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:37:43,496 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:37:44,496 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:37:44,893 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 15:37:45,496 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:37:46,496 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:37:47,502 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:37:48,479 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:37:48,499 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:37:49,499 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:37:50,499 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:37:51,499 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:37:52,391 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 15:37:52,393 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 15:37:52,393 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 15:37:52,394 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 15:37:52,807 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 15:37:52,808 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 15:37:52,814 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:37:53,227 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 15:37:53,805 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:37:53,872 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:37:54,228 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 15:37:54,805 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:37:55,806 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:37:56,806 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:37:57,806 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:37:58,806 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:37:59,542 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:37:59,806 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:38:00,806 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:38:01,806 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:38:02,806 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:38:03,806 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:38:04,806 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:38:05,542 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:38:05,806 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:38:06,807 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:38:07,541 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 15:38:07,542 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 15:38:07,807 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:38:08,807 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:38:09,807 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:38:10,708 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:38:10,807 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:38:11,807 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:38:12,807 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:38:13,807 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:38:14,807 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:38:14,898 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 15:38:15,808 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:38:15,900 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:38:16,808 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:38:17,808 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:38:18,822 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:38:19,813 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:38:20,812 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:38:21,549 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:38:21,810 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:38:22,547 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 15:38:22,551 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 15:38:22,810 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:38:23,810 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:38:24,390 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 15:38:24,391 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 15:38:24,392 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 15:38:24,393 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 15:38:24,811 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:38:25,254 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 15:38:25,811 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:38:26,255 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 15:38:26,811 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:38:27,432 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:38:27,811 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:38:28,811 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:38:29,811 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:38:30,811 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:38:31,811 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:38:32,549 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:38:32,811 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:38:33,811 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:38:34,811 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:38:35,984 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:38:36,983 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:38:37,547 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 15:38:37,548 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 15:38:37,718 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:38:37,983 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:38:38,983 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:38:39,983 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:38:40,984 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:38:41,984 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:38:42,984 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:38:43,549 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:38:43,984 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:38:44,901 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 15:38:44,984 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:38:45,984 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:38:46,984 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:38:47,995 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:38:48,566 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:38:48,994 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:38:49,990 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:38:50,990 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:38:51,990 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:38:52,555 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 15:38:52,555 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 15:38:52,991 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:38:53,145 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 15:38:53,146 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 15:38:53,146 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 15:38:53,147 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 15:38:53,283 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 15:38:53,991 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:38:54,186 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:38:54,284 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 15:38:54,990 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:38:55,990 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:38:56,991 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:38:57,991 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:38:58,991 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:38:59,552 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:38:59,991 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:39:00,991 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:39:01,991 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:39:02,991 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:39:03,991 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:39:04,991 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:39:05,552 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:39:05,991 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:39:06,991 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:39:07,772 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 15:39:07,773 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 15:39:07,992 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:39:08,992 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:39:09,992 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:39:10,885 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:39:10,992 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:39:11,992 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:39:12,992 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:39:13,992 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:39:14,905 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 15:39:14,992 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:39:15,907 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:39:15,992 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:39:16,992 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:39:17,993 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:39:18,993 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:39:19,993 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:39:20,999 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:39:21,687 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:39:21,998 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:39:22,769 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 15:39:22,769 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 15:39:23,001 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:39:23,998 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:39:24,998 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:39:25,998 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:39:26,625 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 15:39:26,627 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 15:39:26,628 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 15:39:26,630 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 15:39:27,031 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:39:27,311 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 15:39:27,667 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:39:28,031 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:39:28,312 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 15:39:29,031 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:39:30,031 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:39:31,031 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:39:32,031 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:39:32,686 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:39:33,031 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:39:34,031 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:39:35,032 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:39:36,032 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:39:37,032 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:39:37,769 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 15:39:37,769 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 15:39:37,847 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:39:38,136 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:39:39,042 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:39:40,042 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:39:41,042 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:39:42,042 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:39:43,042 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:39:43,686 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:39:44,042 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:39:44,908 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 15:39:45,042 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:39:46,043 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:39:47,043 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:39:48,052 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:39:48,688 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:39:49,045 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:39:50,049 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:39:51,051 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:39:52,052 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:39:52,771 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 15:39:52,771 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 15:39:53,049 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:39:53,398 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 15:39:53,399 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 15:39:53,399 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 15:39:53,400 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 15:39:54,050 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:39:54,337 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 15:39:54,439 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:39:55,050 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:39:56,049 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:39:56,338 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 15:39:57,049 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:39:58,049 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:39:59,050 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:39:59,689 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:40:00,050 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:40:01,050 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:40:02,050 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:40:03,050 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:40:04,050 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:40:05,050 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:40:05,689 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:40:06,050 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:40:07,050 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:40:07,771 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 15:40:07,771 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 15:40:08,050 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:40:09,050 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:40:10,051 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:40:10,936 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:40:11,051 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:40:12,051 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:40:13,051 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:40:14,051 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:40:14,913 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 15:40:15,061 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:40:16,059 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:40:16,918 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:40:17,061 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:40:18,059 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:40:19,063 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:40:20,059 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:40:20,749 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 15:40:20,751 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 15:40:20,752 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 15:40:20,753 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 15:40:21,134 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:40:21,370 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 15:40:22,133 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:40:22,371 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 15:40:22,691 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:40:22,771 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 15:40:22,771 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 15:40:23,133 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:40:24,133 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:40:25,133 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:40:26,133 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:40:27,133 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:40:27,692 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:40:28,133 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:40:29,133 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:40:30,133 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:40:31,134 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:40:32,134 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:40:33,189 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:40:33,692 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:40:34,188 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:40:35,189 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:40:36,189 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:40:37,189 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:40:37,771 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 15:40:37,772 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 15:40:38,189 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:40:38,922 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:40:39,189 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:40:40,189 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:40:41,189 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:40:42,189 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:40:43,190 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:40:44,196 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:40:44,695 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:40:44,920 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 15:40:45,196 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:40:46,196 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:40:47,200 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:40:48,200 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:40:49,200 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:40:49,260 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 15:40:49,262 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 15:40:49,262 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 15:40:49,263 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 15:40:49,393 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 15:40:50,200 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:40:50,303 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:40:50,394 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 15:40:51,200 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:40:52,200 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:40:52,771 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 15:40:52,772 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 15:40:53,200 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:40:54,200 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:40:55,200 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:40:55,895 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:40:56,201 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:40:57,201 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:40:58,201 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:40:59,201 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:41:00,201 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:41:01,201 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:41:01,696 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:41:02,201 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:41:03,202 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:41:04,201 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:41:05,201 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:41:06,202 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:41:07,202 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:41:07,695 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:41:07,771 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 15:41:07,772 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 15:41:08,202 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:41:09,202 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:41:10,203 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:41:11,202 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:41:12,202 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:41:13,202 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:41:13,695 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:41:14,202 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:41:14,926 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 15:41:15,206 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:41:16,203 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:41:17,210 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:41:18,203 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:41:18,710 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:41:19,204 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:41:20,204 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:41:20,565 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 15:41:20,566 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 15:41:20,567 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 15:41:20,568 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 15:41:21,204 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:41:21,419 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 15:41:22,204 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:41:22,420 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 15:41:22,771 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 15:41:22,772 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 15:41:23,204 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:41:23,885 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:41:24,204 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:41:25,204 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:41:26,204 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:41:27,204 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:41:28,204 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:41:29,204 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:41:29,701 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:41:30,205 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:41:31,205 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:41:32,205 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:41:33,359 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:41:34,270 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:41:34,701 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:41:35,270 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:41:36,270 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:41:37,271 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:41:37,772 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 15:41:37,772 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 15:41:38,271 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:41:39,271 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:41:39,867 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:41:40,271 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:41:41,271 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:41:42,271 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:41:43,271 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:41:44,271 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:41:44,924 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 15:41:44,926 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:41:45,279 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:41:46,276 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:41:47,277 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:41:48,277 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:41:49,280 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:41:50,277 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:41:50,661 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 15:41:50,663 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 15:41:50,663 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 15:41:50,663 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:41:50,666 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 15:41:51,277 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:41:51,450 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 15:41:52,278 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:41:52,451 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 15:41:52,772 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 15:41:52,772 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 15:41:53,278 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:41:54,278 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:41:55,278 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:41:55,859 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:41:56,278 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:41:57,278 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:41:58,278 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:41:59,278 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:42:00,278 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:42:01,278 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:42:01,704 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:42:02,278 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:42:03,278 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:42:04,305 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:42:05,302 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:42:06,302 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:42:07,302 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:42:07,704 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:42:07,772 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 15:42:07,772 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 15:42:08,302 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:42:09,303 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:42:10,303 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:42:11,303 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:42:12,303 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:42:12,704 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:42:13,303 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:42:14,303 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:42:14,928 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 15:42:15,303 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:42:16,303 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:42:17,303 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:42:17,726 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:42:18,310 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:42:19,310 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:42:20,320 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:42:21,312 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:42:22,312 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:42:22,772 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 15:42:22,773 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 15:42:22,889 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:42:22,992 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 15:42:22,994 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 15:42:22,994 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 15:42:23,000 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 15:42:23,393 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:42:23,475 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 15:42:24,392 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:42:24,475 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 15:42:25,392 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:42:26,392 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:42:27,392 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:42:28,393 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:42:28,710 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:42:29,393 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:42:30,393 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:42:31,393 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:42:32,393 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:42:33,393 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:42:34,393 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:42:34,710 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:42:35,393 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:42:36,393 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:42:37,393 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:42:37,772 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 15:42:37,772 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 15:42:38,393 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:42:39,393 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:42:39,933 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:42:40,394 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:42:41,394 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:42:42,394 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:42:43,394 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:42:44,394 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:42:44,932 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 15:42:44,937 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:42:45,394 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:42:46,394 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:42:47,394 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:42:48,395 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:42:49,402 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:42:50,396 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:42:50,713 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:42:51,404 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:42:52,402 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:42:52,778 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 15:42:52,781 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 15:42:53,402 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:42:54,138 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 15:42:54,139 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 15:42:54,139 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 15:42:54,144 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 15:42:54,504 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 15:42:54,545 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:42:55,544 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:42:56,180 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:42:56,505 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 15:42:56,544 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:42:57,544 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:42:58,544 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:42:59,544 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:43:00,544 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:43:01,544 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:43:01,713 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:43:02,544 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:43:03,544 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:43:04,544 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:43:05,544 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:43:06,545 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:43:07,545 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:43:07,713 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:43:07,778 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 15:43:07,778 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 15:43:08,545 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:43:09,545 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:43:10,545 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:43:11,545 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:43:12,545 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:43:13,545 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:43:13,713 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:43:14,545 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:43:14,936 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 15:43:15,545 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:43:16,545 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:43:17,546 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:43:18,557 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:43:19,552 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:43:19,718 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:43:20,554 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:43:21,556 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:43:22,553 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:43:22,780 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 15:43:22,780 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 15:43:23,485 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 15:43:23,486 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 15:43:23,487 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 15:43:23,488 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 15:43:23,532 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 15:43:23,898 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:43:24,533 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 15:43:24,630 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:43:25,489 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:43:25,635 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:43:26,634 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:43:27,634 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:43:28,634 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:43:29,634 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:43:30,634 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:43:30,716 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:43:31,634 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:43:32,635 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:43:33,635 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:43:34,635 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:43:35,635 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:43:36,635 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:43:36,716 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:43:37,635 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:43:37,780 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 15:43:37,780 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 15:43:38,635 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:43:39,635 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:43:40,635 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:43:41,635 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:43:41,868 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:43:42,636 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:43:43,636 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:43:44,636 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:43:44,941 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 15:43:45,636 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:43:46,636 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:43:46,943 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:43:47,636 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:43:48,649 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:43:49,642 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:43:50,642 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:43:51,645 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:43:52,648 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:43:52,716 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:43:52,790 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 15:43:52,794 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 15:43:53,645 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:43:53,981 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 15:43:53,982 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 15:43:53,982 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 15:43:53,985 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 15:43:54,557 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 15:43:54,645 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:43:55,645 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:43:56,558 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 15:43:56,645 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:43:57,646 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:43:57,717 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:43:58,646 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:43:59,646 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:44:00,646 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:44:01,646 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:44:02,646 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:44:03,646 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:44:03,717 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:44:04,646 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:44:05,646 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:44:06,647 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:44:07,647 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:44:07,783 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 15:44:07,783 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 15:44:08,647 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:44:08,904 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:44:09,647 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:44:10,647 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:44:11,647 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:44:12,647 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:44:13,647 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:44:14,647 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:44:14,717 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:44:14,944 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 15:44:15,647 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:44:16,655 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:44:17,656 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:44:18,651 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:44:19,651 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:44:19,729 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:44:20,653 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:44:21,652 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:44:22,284 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 15:44:22,285 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 15:44:22,286 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 15:44:22,287 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 15:44:22,582 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 15:44:22,684 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:44:22,785 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 15:44:22,786 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 15:44:23,683 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:44:24,584 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 15:44:24,683 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:44:24,864 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:44:25,683 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:44:26,683 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:44:27,683 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:44:28,684 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:44:29,684 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:44:30,684 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:44:30,780 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:44:31,684 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:44:32,684 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:44:33,827 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:44:34,741 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:44:35,741 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:44:36,741 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:44:36,780 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:44:37,741 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:44:37,783 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 15:44:37,784 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 15:44:38,742 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:44:39,742 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:44:40,742 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:44:41,742 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:44:41,864 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:44:42,742 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:44:43,742 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:44:44,753 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:44:44,948 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 15:44:45,748 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:44:46,748 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:44:46,950 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:44:47,750 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:44:48,750 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:44:49,520 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 15:44:49,521 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 15:44:49,522 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 15:44:49,523 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 15:44:49,607 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 15:44:49,921 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:44:50,608 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 15:44:50,918 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:44:51,918 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:44:52,563 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:44:52,783 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 15:44:52,784 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 15:44:52,918 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:44:53,919 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:44:54,918 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:44:55,919 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:44:56,919 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:44:57,780 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:44:57,919 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:44:58,919 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:44:59,919 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:45:00,919 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:45:01,919 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:45:02,919 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:45:03,780 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:45:03,919 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:45:04,920 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:45:05,920 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:45:06,920 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:45:07,784 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 15:45:07,784 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 15:45:07,920 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:45:08,920 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:45:08,924 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:45:09,920 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:45:10,920 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:45:11,920 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:45:12,920 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:45:13,920 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:45:14,781 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:45:14,920 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:45:14,953 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 15:45:15,921 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:45:16,921 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:45:17,928 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:45:18,925 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:45:19,787 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:45:19,928 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:45:20,928 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:45:21,928 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:45:22,784 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 15:45:22,784 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 15:45:22,824 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 15:45:22,925 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 15:45:22,926 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 15:45:22,927 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 15:45:23,216 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:45:23,637 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 15:45:24,214 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:45:24,638 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 15:45:24,928 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:45:25,214 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:45:26,214 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:45:27,214 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:45:28,214 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:45:29,215 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:45:30,215 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:45:30,784 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:45:31,215 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:45:32,215 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:45:33,215 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:45:34,329 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:45:35,328 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:45:36,328 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:45:36,784 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:45:37,328 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:45:37,784 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 15:45:37,785 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 15:45:38,328 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:45:39,329 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:45:40,329 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:45:41,329 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:45:41,936 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:45:42,329 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:45:43,329 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:45:44,329 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:45:44,964 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 15:45:45,330 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:45:46,333 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:45:46,967 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:45:47,333 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:45:48,333 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:45:49,333 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:45:50,201 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 15:45:50,203 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 15:45:50,203 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 15:45:50,205 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 15:45:50,598 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:45:50,667 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 15:45:51,346 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:45:52,243 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:45:52,346 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:45:52,668 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 15:45:52,784 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 15:45:52,785 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 15:45:53,346 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:45:54,346 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:45:55,346 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:45:56,346 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:45:57,347 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:45:57,789 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:45:58,347 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:45:59,347 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:46:00,347 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:46:01,347 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:46:02,347 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:46:02,789 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:46:03,347 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:46:04,347 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:46:05,347 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:46:06,348 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:46:07,348 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:46:07,784 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 15:46:07,785 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 15:46:07,861 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:46:08,348 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:46:09,348 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:46:10,348 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:46:11,348 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:46:12,348 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:46:13,348 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:46:13,789 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:46:14,360 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:46:14,966 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 15:46:15,355 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:46:16,352 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:46:17,352 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:46:18,352 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:46:18,795 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:46:19,353 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:46:19,510 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 15:46:19,511 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 15:46:19,512 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 15:46:19,517 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 15:46:19,700 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 15:46:20,353 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:46:20,702 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 15:46:21,352 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:46:22,353 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:46:22,784 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 15:46:22,785 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 15:46:23,353 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:46:23,899 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:46:24,353 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:46:25,353 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:46:26,353 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:46:27,353 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:46:28,353 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:46:29,353 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:46:29,793 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:46:30,353 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:46:31,353 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:46:32,353 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:46:33,354 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:46:34,354 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:46:35,027 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:46:35,354 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:46:36,354 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:46:37,354 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:46:37,784 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 15:46:37,785 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 15:46:38,354 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:46:39,354 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:46:40,354 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:46:40,940 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:46:41,354 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:46:42,354 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:46:43,354 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:46:44,355 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:46:44,967 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 15:46:45,360 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:46:45,970 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:46:46,362 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:46:47,363 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:46:48,363 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:46:49,363 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:46:50,363 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:46:50,922 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 15:46:50,924 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 15:46:50,924 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 15:46:50,930 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 15:46:51,363 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:46:51,729 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 15:46:51,964 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:46:52,363 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:46:52,730 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 15:46:52,785 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 15:46:52,785 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 15:46:53,363 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:46:54,364 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:46:55,364 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:46:56,364 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:46:57,364 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:46:57,940 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:46:58,364 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:46:59,364 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:47:00,364 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:47:01,364 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:47:02,364 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:47:03,364 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:47:03,940 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:47:04,365 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:47:05,365 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:47:06,365 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:47:07,365 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:47:07,785 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 15:47:07,785 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 15:47:08,365 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:47:09,365 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:47:09,940 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:47:10,365 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:47:11,365 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:47:12,365 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:47:13,365 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:47:14,365 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:47:14,969 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 15:47:14,972 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:47:15,366 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:47:16,377 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:47:17,375 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:47:18,375 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:47:19,375 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:47:20,375 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:47:20,949 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:47:21,375 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:47:21,394 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 15:47:21,396 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 15:47:21,396 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 15:47:21,399 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 15:47:21,755 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 15:47:22,376 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:47:22,756 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 15:47:22,785 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 15:47:22,785 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 15:47:23,376 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:47:24,376 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:47:25,376 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:47:26,376 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:47:26,946 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:47:27,376 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:47:28,376 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:47:29,376 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:47:30,376 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:47:31,376 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:47:32,376 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:47:32,946 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:47:33,376 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:47:34,377 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:47:35,377 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:47:36,377 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:47:37,377 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:47:37,785 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 15:47:37,786 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 15:47:37,946 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:47:38,377 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:47:39,377 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:47:40,377 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:47:41,377 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:47:42,377 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:47:42,946 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:47:43,377 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:47:44,377 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:47:44,973 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 15:47:45,377 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:47:46,386 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:47:47,382 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:47:47,949 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:47:48,384 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:47:49,389 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:47:50,385 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:47:51,382 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:47:51,979 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 15:47:51,980 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 15:47:51,981 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 15:47:51,984 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 15:47:52,385 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:47:52,782 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 15:47:52,782 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 15:47:52,785 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 15:47:52,786 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 15:47:52,949 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:47:53,384 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:47:54,384 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:47:55,384 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:47:56,384 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:47:57,384 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:47:57,950 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:47:58,384 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:47:59,384 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:48:00,384 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:48:01,384 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:48:02,384 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:48:02,950 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:48:03,384 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:48:04,385 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:48:05,485 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:48:06,482 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:48:07,482 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:48:07,785 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 15:48:07,786 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 15:48:07,952 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:48:08,483 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:48:09,483 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:48:10,483 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:48:11,483 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:48:12,483 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:48:13,483 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:48:13,950 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:48:14,483 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:48:14,978 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 15:48:15,483 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:48:16,483 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:48:17,484 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:48:18,491 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:48:18,956 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:48:19,495 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:48:20,492 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:48:21,489 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:48:22,492 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:48:22,785 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 15:48:22,786 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 15:48:23,490 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:48:23,595 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 15:48:23,596 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 15:48:23,597 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 15:48:23,599 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 15:48:23,816 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 15:48:24,490 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:48:24,637 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:48:24,817 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 15:48:25,489 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:48:26,490 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:48:27,490 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:48:28,490 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:48:29,490 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:48:29,954 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:48:30,490 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:48:31,490 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:48:32,490 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:48:33,490 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:48:34,490 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:48:35,491 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:48:35,954 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:48:36,492 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:48:37,491 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:48:37,786 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 15:48:37,786 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 15:48:38,491 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:48:39,491 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:48:40,491 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:48:40,954 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:48:41,491 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:48:42,491 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:48:43,491 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:48:44,491 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:48:44,981 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 15:48:45,491 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:48:45,984 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:48:46,491 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:48:47,492 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:48:48,503 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:48:49,498 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:48:50,498 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:48:51,501 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:48:51,976 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:48:52,498 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:48:52,788 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 15:48:52,788 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 15:48:53,372 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 15:48:53,374 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 15:48:53,374 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 15:48:53,376 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 15:48:53,767 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:48:53,849 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 15:48:54,517 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:48:54,850 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 15:48:55,517 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:48:56,517 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:48:57,414 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:48:57,517 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:48:58,518 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:48:59,518 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:49:00,518 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:49:01,518 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:49:02,518 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:49:02,960 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:49:03,518 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:49:04,518 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:49:05,518 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:49:06,518 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:49:07,519 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:49:07,788 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 15:49:07,789 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 15:49:07,960 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:49:08,519 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:49:09,519 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:49:10,519 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:49:11,519 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:49:12,519 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:49:12,960 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:49:13,519 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:49:14,519 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:49:14,986 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 15:49:15,519 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:49:16,519 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:49:17,519 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:49:18,520 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:49:18,965 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:49:19,529 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:49:20,527 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:49:21,527 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:49:22,532 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:49:22,791 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 15:49:22,791 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 15:49:23,527 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:49:23,965 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:49:24,349 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 15:49:24,351 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 15:49:24,351 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 15:49:24,352 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 15:49:24,752 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:49:24,875 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 15:49:25,751 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:49:26,751 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:49:26,876 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 15:49:27,751 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:49:28,751 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:49:29,752 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:49:29,963 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:49:30,752 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:49:31,752 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:49:32,752 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:49:33,752 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:49:34,752 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:49:35,752 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:49:35,963 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:49:36,752 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:49:37,753 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:49:37,791 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 15:49:37,792 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 15:49:38,753 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:49:39,752 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:49:40,753 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:49:40,963 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:49:41,753 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:49:42,753 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:49:43,753 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:49:44,753 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:49:44,990 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 15:49:45,753 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:49:45,992 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:49:46,753 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:49:47,753 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:49:48,754 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:49:49,759 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:49:50,754 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:49:51,759 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:49:51,968 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:49:52,763 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:49:52,795 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 15:49:52,796 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 15:49:53,759 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:49:54,691 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 15:49:54,692 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 15:49:54,693 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 15:49:54,694 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 15:49:54,903 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 15:49:55,119 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:49:55,837 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:49:56,837 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:49:56,905 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 15:49:57,733 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:49:57,837 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:49:58,837 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:49:59,837 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:50:00,838 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:50:01,838 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:50:02,838 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:50:02,967 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:50:03,838 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:50:04,838 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:50:05,838 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:50:06,838 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:50:07,793 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 15:50:07,794 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 15:50:07,838 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:50:08,839 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:50:08,967 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:50:09,838 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:50:10,839 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:50:11,839 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:50:12,839 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:50:13,839 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:50:13,968 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:50:14,839 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:50:14,992 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 15:50:15,839 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:50:16,839 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:50:17,839 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:50:18,848 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:50:18,970 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:50:19,845 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:50:20,846 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:50:21,848 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:50:22,793 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 15:50:22,793 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 15:50:22,846 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:50:23,631 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 15:50:23,632 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 15:50:23,632 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 15:50:23,633 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 15:50:23,928 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 15:50:24,039 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:50:24,673 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:50:24,929 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 15:50:25,038 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:50:26,038 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:50:27,038 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:50:28,038 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:50:29,038 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:50:29,969 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:50:30,038 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:50:31,038 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:50:32,038 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:50:33,039 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:50:34,039 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:50:35,039 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:50:35,969 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:50:36,039 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:50:37,039 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:50:37,793 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 15:50:37,794 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 15:50:38,039 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:50:39,039 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:50:40,039 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:50:41,039 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:50:41,969 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:50:42,040 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:50:43,039 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:50:44,040 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:50:44,997 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 15:50:45,040 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:50:46,058 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:50:47,005 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:50:47,050 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:50:48,047 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:50:49,047 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:50:50,047 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:50:51,039 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 15:50:51,040 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 15:50:51,041 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 15:50:51,042 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 15:50:51,441 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:50:51,962 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 15:50:52,080 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:50:52,440 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:50:52,793 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 15:50:52,794 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 15:50:52,963 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 15:50:53,440 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:50:54,440 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:50:55,440 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:50:56,440 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:50:57,440 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:50:57,973 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:50:58,440 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:50:59,440 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:51:00,440 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:51:01,440 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:51:02,441 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:51:03,441 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:51:03,973 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:51:04,441 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:51:05,441 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:51:06,441 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:51:07,441 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:51:07,793 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 15:51:07,794 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 15:51:08,441 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:51:09,441 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:51:09,973 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:51:10,441 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:51:11,441 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:51:12,442 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:51:13,442 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:51:14,442 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:51:14,973 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:51:15,000 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 15:51:15,442 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:51:16,442 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:51:17,452 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:51:18,447 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:51:19,448 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:51:19,979 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:51:20,445 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:51:21,447 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:51:22,447 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:51:22,677 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 15:51:22,679 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 15:51:22,679 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 15:51:22,681 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 15:51:23,003 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 15:51:23,082 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 15:51:23,083 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 15:51:23,447 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:51:24,447 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:51:25,004 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 15:51:25,244 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:51:25,447 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:51:26,447 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:51:27,447 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:51:28,447 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:51:29,448 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:51:30,448 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:51:31,078 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:51:31,448 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:51:32,448 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:51:33,448 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:51:34,448 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:51:35,448 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:51:36,448 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:51:37,078 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:51:37,448 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:51:37,823 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 15:51:37,823 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 15:51:38,448 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:51:39,449 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:51:40,449 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:51:41,449 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:51:42,449 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:51:43,078 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:51:43,449 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:51:44,449 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:51:45,005 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 15:51:45,449 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:51:46,449 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:51:47,456 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:51:48,088 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:51:48,457 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:51:49,456 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:51:50,459 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:51:51,454 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:51:52,254 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 15:51:52,255 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 15:51:52,255 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 15:51:52,257 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 15:51:52,685 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:51:52,823 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 15:51:52,824 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 15:51:53,028 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 15:51:53,029 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 15:51:53,684 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:51:54,083 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:51:54,684 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:51:55,684 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:51:56,684 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:51:57,685 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:51:58,685 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:51:59,685 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:52:00,083 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:52:00,685 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:52:01,685 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:52:02,685 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:52:03,685 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:52:04,685 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:52:05,686 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:52:06,083 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:52:06,685 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:52:07,685 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:52:07,823 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 15:52:07,823 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 15:52:08,686 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:52:09,686 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:52:10,686 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:52:11,083 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:52:11,686 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:52:12,686 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:52:13,686 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:52:14,686 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:52:15,009 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 15:52:15,686 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:52:16,686 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:52:17,011 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:52:17,701 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:52:18,699 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:52:19,696 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:52:20,693 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:52:21,694 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:52:22,088 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:52:22,693 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:52:22,823 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 15:52:22,824 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 15:52:23,039 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 15:52:23,041 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 15:52:23,042 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 15:52:23,043 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 15:52:23,053 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 15:52:23,693 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:52:24,693 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:52:25,054 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 15:52:25,694 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:52:26,694 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:52:27,483 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:52:27,694 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:52:28,694 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:52:29,694 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:52:30,694 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:52:31,694 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:52:32,694 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:52:33,187 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:52:33,694 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:52:34,694 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:52:35,694 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:52:36,694 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:52:37,695 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:52:37,961 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 15:52:37,962 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 15:52:38,188 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:52:38,695 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:52:39,695 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:52:40,695 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:52:41,695 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:52:42,695 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:52:43,188 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:52:43,695 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:52:44,695 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:52:45,013 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 15:52:45,695 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:52:46,696 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:52:47,696 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:52:48,696 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:52:49,188 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:52:49,696 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:52:50,702 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:52:51,700 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:52:52,705 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:52:52,871 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 15:52:52,874 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 15:52:53,702 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:52:54,189 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:52:54,702 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:52:55,704 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:52:56,342 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 15:52:56,344 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 15:52:56,344 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 15:52:56,345 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 15:52:56,775 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:52:57,080 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 15:52:57,081 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 15:52:57,772 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:52:58,772 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:52:59,190 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:52:59,772 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:53:00,772 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:53:01,772 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:53:02,772 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:53:03,772 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:53:04,773 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:53:05,190 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:53:05,773 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:53:06,773 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:53:07,773 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:53:07,871 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 15:53:07,871 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 15:53:08,773 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:53:09,773 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:53:10,773 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:53:11,190 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:53:11,773 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:53:12,774 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:53:13,773 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:53:14,774 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:53:15,017 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 15:53:15,774 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:53:16,774 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:53:17,019 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:53:17,774 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:53:18,774 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:53:19,774 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:53:20,780 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:53:21,785 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:53:22,193 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:53:22,782 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:53:22,873 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 15:53:22,875 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 15:53:23,781 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:53:24,788 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:53:25,785 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:53:26,337 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 15:53:26,338 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 15:53:26,338 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 15:53:26,339 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 15:53:26,785 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:53:27,107 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 15:53:27,108 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 15:53:27,378 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:53:27,785 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:53:28,785 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:53:29,785 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:53:30,785 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:53:31,785 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:53:32,785 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:53:33,196 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:53:33,785 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:53:34,786 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:53:35,786 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:53:36,786 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:53:37,786 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:53:38,096 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 15:53:38,097 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 15:53:38,196 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:53:38,786 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:53:39,786 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:53:40,786 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:53:41,787 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:53:42,786 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:53:43,196 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:53:43,787 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:53:44,787 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:53:45,021 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 15:53:45,787 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:53:46,787 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:53:47,787 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:53:48,787 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:53:49,196 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:53:49,801 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:53:50,791 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:53:51,791 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:53:52,793 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:53:53,005 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 15:53:53,008 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 15:53:53,791 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:53:54,200 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:53:54,791 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:53:54,885 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 15:53:54,887 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 15:53:54,887 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 15:53:54,888 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 15:53:55,137 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 15:53:55,791 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:53:56,791 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:53:57,139 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 15:53:57,791 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:53:58,792 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:53:59,201 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:53:59,792 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:54:00,792 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:54:01,792 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:54:02,792 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:54:03,792 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:54:04,792 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:54:05,201 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:54:05,792 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:54:06,792 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:54:07,793 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:54:08,003 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 15:54:08,004 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 15:54:08,792 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:54:09,793 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:54:10,201 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:54:10,793 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:54:11,793 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:54:12,793 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:54:13,793 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:54:14,793 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:54:15,025 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 15:54:15,793 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:54:16,027 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:54:16,793 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:54:17,793 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:54:18,793 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:54:19,796 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:54:20,801 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:54:21,209 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:54:21,802 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:54:22,807 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:54:23,009 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 15:54:23,010 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 15:54:23,802 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:54:24,704 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 15:54:24,704 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 15:54:24,705 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 15:54:24,707 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 15:54:25,099 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:54:25,168 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 15:54:26,099 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:54:26,745 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:54:27,099 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:54:27,170 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 15:54:28,099 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:54:29,100 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:54:30,099 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:54:31,099 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:54:32,099 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:54:32,208 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:54:33,099 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:54:34,099 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:54:35,100 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:54:36,100 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:54:37,100 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:54:38,006 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 15:54:38,007 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 15:54:38,100 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:54:38,109 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:54:39,100 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:54:40,100 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:54:41,100 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:54:42,100 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:54:43,100 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:54:43,208 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:54:44,100 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:54:45,029 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 15:54:45,100 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:54:46,101 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:54:47,108 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:54:48,106 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:54:48,209 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:54:49,109 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:54:50,110 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:54:51,110 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:54:52,110 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:54:52,313 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 15:54:52,314 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 15:54:52,315 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 15:54:52,316 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 15:54:53,006 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 15:54:53,007 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 15:54:53,111 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:54:53,192 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 15:54:53,193 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 15:54:54,110 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:54:54,208 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:54:55,110 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:54:56,110 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:54:57,110 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:54:58,110 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:54:59,111 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:55:00,111 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:55:00,208 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:55:01,111 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:55:02,111 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:55:03,111 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:55:04,111 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:55:05,111 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:55:05,208 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:55:06,111 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:55:07,132 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:55:08,006 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 15:55:08,007 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 15:55:08,130 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:55:09,130 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:55:10,130 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:55:11,130 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:55:11,209 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:55:12,131 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:55:13,131 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:55:14,131 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:55:15,033 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 15:55:15,131 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:55:16,131 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:55:17,035 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:55:17,131 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:55:18,131 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:55:19,141 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:55:20,136 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:55:21,137 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:55:22,136 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:55:22,220 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:55:23,010 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 15:55:23,010 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 15:55:23,139 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:55:24,139 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:55:24,214 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 15:55:24,215 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 15:55:24,216 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 15:55:24,217 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 15:55:24,219 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 15:55:25,145 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:55:25,220 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 15:55:26,139 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:55:27,139 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:55:27,257 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:55:28,139 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:55:29,139 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:55:30,139 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:55:31,140 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:55:32,140 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:55:33,140 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:55:33,213 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:55:34,140 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:55:35,140 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:55:36,140 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:55:37,140 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:55:38,010 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 15:55:38,011 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 15:55:38,140 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:55:39,140 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:55:39,213 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:55:40,141 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:55:41,141 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:55:42,141 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:55:43,141 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:55:44,141 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:55:44,213 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:55:45,037 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 15:55:45,141 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:55:46,141 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:55:47,141 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:55:48,141 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:55:49,152 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:55:49,215 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:55:50,149 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:55:51,149 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:55:52,147 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:55:53,015 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 15:55:53,023 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 15:55:53,147 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:55:54,151 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:55:54,216 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:55:54,594 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 15:55:54,595 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 15:55:54,596 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 15:55:54,597 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 15:55:55,148 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:55:55,249 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 15:55:55,249 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 15:55:56,147 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:55:57,148 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:55:58,148 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:55:59,148 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:55:59,217 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:56:00,148 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:56:01,148 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:56:02,148 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:56:03,148 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:56:04,148 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:56:05,148 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:56:05,217 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:56:06,148 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:56:07,148 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:56:08,014 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 15:56:08,015 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 15:56:08,149 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:56:09,149 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:56:10,149 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:56:11,149 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:56:11,217 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:56:12,149 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:56:13,149 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:56:14,149 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:56:15,041 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 15:56:15,149 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:56:16,149 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:56:17,043 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:56:17,149 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:56:18,150 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:56:19,150 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:56:20,150 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:56:21,163 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:56:22,154 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:56:22,228 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:56:23,021 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 15:56:23,022 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 15:56:23,155 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:56:24,154 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:56:25,075 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 15:56:25,076 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 15:56:25,076 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 15:56:25,078 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 15:56:25,273 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 15:56:25,273 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 15:56:25,482 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:56:26,481 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:56:27,481 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:56:28,118 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:56:28,481 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:56:29,481 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:56:30,481 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:56:31,481 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:56:32,481 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:56:33,218 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:56:33,482 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:56:34,482 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:56:35,482 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:56:36,482 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:56:37,482 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:56:38,018 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 15:56:38,018 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 15:56:38,535 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:56:39,218 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:56:39,534 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:56:40,534 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:56:41,534 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:56:42,534 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:56:43,534 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:56:44,218 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:56:44,535 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:56:45,045 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 15:56:45,535 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:56:46,535 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:56:47,535 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:56:48,535 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:56:49,535 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:56:50,218 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:56:50,535 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:56:51,535 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:56:52,541 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:56:53,023 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 15:56:53,027 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 15:56:53,541 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:56:54,541 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:56:55,229 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:56:55,541 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:56:56,541 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:56:57,529 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 15:56:57,530 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 15:56:57,531 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 15:56:57,532 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 15:56:57,950 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:56:58,304 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 15:56:58,680 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:56:59,680 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:57:00,305 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 15:57:00,680 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:57:01,222 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:57:01,680 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:57:02,680 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:57:03,680 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:57:04,680 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:57:05,680 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:57:06,680 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:57:07,222 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:57:07,680 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:57:08,023 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 15:57:08,024 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 15:57:08,680 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:57:09,681 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:57:10,915 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:57:11,774 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:57:12,774 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:57:13,222 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:57:13,775 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:57:14,775 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:57:15,050 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 15:57:15,775 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:57:16,775 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:57:17,775 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:57:18,775 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:57:19,222 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:57:19,775 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:57:20,775 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:57:21,775 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:57:22,775 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:57:23,023 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 15:57:23,024 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 15:57:23,784 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:57:24,225 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:57:24,777 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:57:25,777 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:57:26,777 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:57:27,777 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:57:28,777 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:57:29,229 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:57:29,277 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 15:57:29,279 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 15:57:29,279 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 15:57:29,280 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 15:57:29,323 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 15:57:29,778 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:57:30,324 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 15:57:30,782 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:57:31,781 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:57:32,781 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:57:33,781 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:57:34,781 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:57:35,226 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:57:35,781 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:57:36,781 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:57:37,782 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:57:38,023 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 15:57:38,024 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 15:57:38,782 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:57:39,782 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:57:40,782 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:57:41,226 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:57:41,828 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:57:42,820 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:57:43,820 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:57:44,820 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:57:45,053 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 15:57:45,820 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:57:46,820 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:57:47,055 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:57:47,821 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:57:48,821 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:57:49,821 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:57:50,821 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:57:51,821 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:57:52,226 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:57:52,821 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:57:53,023 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 15:57:53,024 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 15:57:53,831 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:57:54,825 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:57:55,825 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:57:56,825 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:57:57,233 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:57:57,825 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:57:58,826 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:57:59,433 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 15:57:59,434 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 15:57:59,435 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 15:57:59,437 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 15:57:59,837 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:58:00,353 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 15:58:00,836 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:58:01,836 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:58:02,354 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 15:58:02,475 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:58:02,836 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:58:03,836 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:58:04,836 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:58:05,836 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:58:06,837 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:58:07,837 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:58:08,023 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 15:58:08,024 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 15:58:08,174 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:58:08,837 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:58:09,837 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:58:10,837 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:58:11,837 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:58:12,837 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:58:13,230 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:58:13,837 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:58:14,837 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:58:15,057 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 15:58:15,838 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:58:16,838 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:58:17,838 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:58:18,838 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:58:19,230 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:58:19,838 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:58:20,838 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:58:21,838 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:58:22,838 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:58:23,023 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 15:58:23,024 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 15:58:23,839 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:58:24,842 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:58:25,235 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:58:25,842 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:58:26,846 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:58:27,842 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:58:28,842 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:58:29,842 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:58:30,237 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:58:30,295 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 15:58:30,297 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 15:58:30,298 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 15:58:30,299 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 15:58:30,376 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 15:58:30,843 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:58:31,843 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:58:32,378 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 15:58:32,842 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:58:33,843 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:58:34,843 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:58:35,843 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:58:36,234 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:58:36,843 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:58:37,843 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:58:38,024 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 15:58:38,024 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 15:58:38,843 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:58:39,843 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:58:40,843 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:58:41,843 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:58:42,234 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:58:43,050 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:58:44,050 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:58:45,050 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:58:45,061 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 15:58:46,050 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:58:47,050 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:58:48,050 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:58:48,063 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:58:49,050 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:58:50,050 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:58:51,050 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:58:52,050 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:58:53,032 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 15:58:53,040 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 15:58:53,071 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:58:53,149 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:58:54,055 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:58:55,057 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:58:56,060 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:58:57,063 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:58:58,058 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:58:58,238 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:58:58,447 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 15:58:58,449 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 15:58:58,449 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 15:58:58,451 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 15:58:59,058 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:58:59,403 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 15:59:00,058 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:59:00,404 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 15:59:01,058 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:59:02,059 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:59:03,058 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:59:04,059 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:59:04,238 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:59:05,059 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:59:06,059 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:59:07,059 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:59:08,026 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 15:59:08,027 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 15:59:08,066 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:59:09,059 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:59:10,059 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:59:10,238 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:59:11,060 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:59:12,060 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:59:13,059 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:59:14,060 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:59:15,060 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:59:15,065 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 15:59:16,060 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:59:16,067 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:59:17,060 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:59:18,060 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:59:19,060 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:59:20,060 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:59:21,060 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:59:21,239 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:59:22,060 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:59:23,034 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 15:59:23,037 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 15:59:23,074 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:59:24,064 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:59:25,064 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:59:26,064 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:59:26,252 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:59:27,066 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:59:28,064 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:59:28,555 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 15:59:28,556 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 15:59:28,557 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 15:59:28,558 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 15:59:29,065 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:59:29,440 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 15:59:30,064 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:59:30,441 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 15:59:31,064 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:59:31,596 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:59:32,065 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:59:33,065 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:59:34,065 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:59:35,065 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:59:36,065 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:59:37,065 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:59:37,241 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:59:38,031 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 15:59:38,031 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 15:59:38,071 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:59:39,065 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:59:40,127 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:59:41,126 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:59:42,126 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:59:43,126 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:59:43,241 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:59:44,126 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:59:45,069 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 15:59:45,126 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:59:46,126 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:59:47,126 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:59:48,126 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:59:49,126 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:59:49,241 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:59:50,127 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:59:51,127 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:59:52,135 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:59:53,032 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 15:59:53,038 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 15:59:53,131 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:59:54,131 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:59:54,247 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 15:59:55,133 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:59:56,131 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:59:57,131 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:59:57,537 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 15:59:57,539 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 15:59:57,539 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 15:59:57,541 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 15:59:58,135 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 15:59:58,468 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 15:59:59,135 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:00:00,135 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:00:00,246 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:00:00,470 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 16:00:01,135 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:00:02,135 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:00:03,135 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:00:04,135 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:00:05,135 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:00:06,135 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:00:06,246 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:00:07,135 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:00:08,032 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 16:00:08,032 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 16:00:08,135 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:00:09,136 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:00:10,314 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:00:11,313 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:00:12,246 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:00:12,313 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:00:13,313 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:00:14,313 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:00:15,072 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 16:00:15,313 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:00:16,313 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:00:17,313 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:00:18,073 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:00:18,313 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:00:19,314 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:00:20,314 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:00:21,314 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:00:22,314 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:00:23,039 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 16:00:23,047 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 16:00:23,154 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:00:23,317 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:00:24,317 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:00:25,318 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:00:26,319 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:00:27,318 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:00:28,114 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 16:00:28,116 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 16:00:28,117 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 16:00:28,118 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 16:00:28,155 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:00:28,494 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 16:00:28,515 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:00:29,513 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:00:30,496 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 16:00:30,513 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:00:31,513 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:00:32,513 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:00:33,259 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:00:33,513 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:00:34,513 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:00:35,513 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:00:36,513 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:00:37,513 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:00:38,034 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 16:00:38,034 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 16:00:38,513 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:00:39,259 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:00:39,513 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:00:40,514 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:00:41,514 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:00:42,514 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:00:43,514 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:00:44,260 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:00:44,514 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:00:45,076 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 16:00:45,514 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:00:46,514 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:00:47,514 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:00:48,514 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:00:49,514 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:00:50,260 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:00:50,514 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:00:51,515 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:00:52,515 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:00:53,034 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 16:00:53,034 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 16:00:53,515 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:00:54,515 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:00:55,260 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:00:55,515 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:00:56,527 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:00:57,520 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:00:58,521 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:00:59,525 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:01:00,263 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:01:00,522 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:01:01,522 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:01:01,622 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 16:01:01,624 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 16:01:01,624 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 16:01:01,626 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 16:01:02,521 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 16:01:02,522 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:01:03,522 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:01:04,522 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 16:01:04,522 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:01:05,264 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:01:05,522 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:01:06,522 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:01:07,522 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:01:08,034 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 16:01:08,035 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 16:01:08,522 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:01:09,523 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:01:10,523 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:01:11,264 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:01:11,523 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:01:12,523 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:01:13,523 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:01:14,523 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:01:15,080 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 16:01:15,579 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:01:16,578 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:01:17,082 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:01:17,578 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:01:18,578 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:01:19,578 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:01:20,579 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:01:21,579 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:01:22,264 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:01:22,579 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:01:23,034 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 16:01:23,035 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 16:01:23,579 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:01:24,579 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:01:25,579 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:01:26,579 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:01:27,269 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:01:27,590 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:01:28,583 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:01:29,583 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:01:30,583 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:01:31,587 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:01:32,587 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:01:32,933 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 16:01:32,934 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 16:01:32,935 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 16:01:32,935 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:01:32,936 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 16:01:33,546 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 16:01:33,587 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:01:34,547 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 16:01:34,587 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:01:35,587 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:01:36,587 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:01:37,587 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:01:38,034 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 16:01:38,035 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 16:01:38,199 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:01:38,587 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:01:39,588 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:01:40,588 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:01:41,588 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:01:42,588 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:01:43,325 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:01:43,588 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:01:44,771 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:01:45,084 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 16:01:45,684 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:01:46,684 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:01:47,684 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:01:48,325 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:01:48,684 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:01:49,684 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:01:50,684 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:01:51,684 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:01:52,684 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:01:53,034 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 16:01:53,035 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 16:01:53,684 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:01:54,325 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:01:54,684 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:01:55,690 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:01:56,686 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:01:57,687 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:01:58,691 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:01:59,337 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:01:59,693 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:02:00,691 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:02:01,295 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 16:02:01,297 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 16:02:01,297 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 16:02:01,299 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 16:02:01,572 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 16:02:01,698 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:02:02,573 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 16:02:02,694 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:02:03,694 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:02:04,694 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:02:05,333 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:02:05,694 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:02:06,694 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:02:07,694 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:02:08,035 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 16:02:08,035 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 16:02:08,694 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:02:09,694 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:02:10,694 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:02:11,333 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:02:11,695 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:02:12,695 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:02:13,695 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:02:14,695 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:02:15,088 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 16:02:15,695 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:02:16,695 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:02:17,090 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:02:17,695 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:02:18,695 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:02:19,695 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:02:20,695 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:02:21,696 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:02:22,333 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:02:22,696 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:02:23,035 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 16:02:23,035 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 16:02:23,696 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:02:24,696 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:02:25,696 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:02:26,696 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:02:27,343 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:02:27,699 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:02:28,705 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:02:29,702 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:02:30,702 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:02:31,702 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:02:32,341 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:02:32,492 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 16:02:32,494 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 16:02:32,494 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 16:02:32,496 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 16:02:32,602 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 16:02:32,906 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:02:33,903 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:02:34,604 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 16:02:34,903 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:02:35,903 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:02:36,903 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:02:37,904 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:02:38,035 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 16:02:38,035 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 16:02:38,204 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:02:38,903 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:02:39,904 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:02:40,904 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:02:41,904 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:02:42,904 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:02:43,336 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:02:43,904 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:02:44,904 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:02:45,092 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 16:02:45,904 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:02:46,904 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:02:47,905 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:02:48,336 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:02:48,905 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:02:49,905 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:02:50,905 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:02:51,905 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:02:52,905 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:02:53,035 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 16:02:53,035 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 16:02:53,905 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:02:54,336 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:02:54,914 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:02:55,908 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:02:56,909 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:02:57,911 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:02:58,916 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:02:59,339 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:02:59,909 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:03:00,117 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 16:03:00,119 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 16:03:00,119 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 16:03:00,121 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 16:03:00,630 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 16:03:00,910 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:03:01,912 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:03:02,631 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 16:03:02,910 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:03:03,910 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:03:04,340 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:03:04,910 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:03:05,910 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:03:06,910 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:03:07,910 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:03:08,035 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 16:03:08,035 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 16:03:08,910 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:03:09,910 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:03:10,340 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:03:10,910 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:03:11,910 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:03:12,910 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:03:13,911 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:03:14,911 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:03:15,096 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 16:03:15,911 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:03:16,098 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:03:16,911 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:03:17,911 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:03:18,911 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:03:19,911 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:03:20,911 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:03:21,359 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:03:21,911 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:03:22,912 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:03:23,035 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 16:03:23,036 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 16:03:23,912 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:03:24,912 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:03:25,912 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:03:26,364 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:03:26,916 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:03:27,916 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:03:28,916 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:03:29,917 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:03:30,917 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:03:31,366 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:03:31,859 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 16:03:31,860 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 16:03:31,860 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 16:03:31,861 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 16:03:32,262 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:03:32,659 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 16:03:33,006 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:03:34,006 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:03:34,661 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 16:03:35,006 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:03:36,006 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:03:37,006 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:03:37,362 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:03:38,007 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:03:38,035 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 16:03:38,036 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 16:03:39,007 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:03:40,007 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:03:41,007 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:03:42,007 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:03:43,007 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:03:43,362 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:03:44,008 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:03:45,007 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:03:45,100 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 16:03:46,008 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:03:47,008 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:03:48,008 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:03:48,362 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:03:49,008 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:03:50,008 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:03:51,008 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:03:52,008 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:03:53,008 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:03:53,035 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 16:03:53,036 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 16:03:54,008 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:03:54,362 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:03:55,009 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:03:56,012 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:03:57,016 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:03:58,015 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:03:59,012 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:03:59,370 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:04:00,012 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:04:00,965 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 16:04:00,966 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 16:04:00,966 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 16:04:00,967 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 16:04:01,364 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:04:01,686 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 16:04:02,111 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:04:02,687 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 16:04:03,111 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:04:04,112 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:04:05,112 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:04:05,364 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:04:06,112 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:04:07,112 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:04:08,035 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 16:04:08,036 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 16:04:08,112 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:04:09,112 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:04:10,112 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:04:10,364 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:04:11,112 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:04:12,112 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:04:13,112 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:04:14,113 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:04:15,104 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 16:04:15,113 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:04:16,106 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:04:16,113 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:04:17,113 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:04:18,113 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:04:19,113 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:04:20,113 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:04:21,113 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:04:21,364 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:04:22,113 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:04:23,036 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 16:04:23,036 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 16:04:23,113 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:04:24,113 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:04:25,114 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:04:26,118 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:04:26,366 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:04:27,118 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:04:28,118 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:04:29,121 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:04:30,123 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:04:31,122 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:04:31,371 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 16:04:31,372 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 16:04:31,373 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 16:04:31,373 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:04:31,374 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 16:04:31,719 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 16:04:32,122 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:04:32,720 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 16:04:33,122 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:04:34,122 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:04:35,122 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:04:36,122 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:04:37,122 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:04:37,370 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:04:38,036 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 16:04:38,036 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 16:04:38,123 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:04:39,122 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:04:40,122 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:04:41,123 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:04:42,123 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:04:43,123 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:04:43,370 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:04:44,123 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:04:45,108 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 16:04:45,123 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:04:46,123 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:04:47,123 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:04:48,123 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:04:49,123 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:04:49,370 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:04:50,123 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:04:51,123 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:04:52,124 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:04:53,036 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 16:04:53,036 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 16:04:53,124 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:04:54,124 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:04:54,370 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:04:55,124 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:04:56,124 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:04:57,124 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:04:58,130 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:04:59,127 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:04:59,386 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:05:00,127 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:05:01,127 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:05:02,128 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:05:03,128 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:05:03,161 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 16:05:03,162 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 16:05:03,163 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 16:05:03,164 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 16:05:03,749 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 16:05:04,127 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:05:04,563 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:05:04,750 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 16:05:05,127 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:05:06,128 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:05:07,128 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:05:08,036 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 16:05:08,037 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 16:05:08,128 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:05:09,128 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:05:10,128 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:05:10,562 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:05:11,128 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:05:12,128 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:05:13,128 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:05:14,128 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:05:15,112 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 16:05:15,128 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:05:16,114 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:05:16,129 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:05:17,201 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:05:18,200 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:05:19,200 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:05:20,201 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:05:21,201 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:05:21,562 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:05:22,201 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:05:23,036 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 16:05:23,037 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 16:05:23,201 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:05:24,201 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:05:25,201 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:05:26,201 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:05:27,201 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:05:27,562 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:05:28,201 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:05:29,202 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:05:30,210 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:05:31,214 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:05:32,207 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:05:32,569 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:05:33,214 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:05:34,207 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:05:35,208 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:05:35,377 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 16:05:35,378 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 16:05:35,379 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 16:05:35,381 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 16:05:35,780 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 16:05:36,214 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:05:36,780 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 16:05:37,208 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:05:38,037 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 16:05:38,037 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 16:05:38,158 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:05:38,208 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:05:39,208 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:05:40,208 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:05:41,208 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:05:42,208 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:05:43,208 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:05:43,565 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:05:44,208 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:05:45,116 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 16:05:45,208 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:05:46,208 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:05:47,360 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:05:48,357 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:05:48,565 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:05:49,357 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:05:50,357 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:05:51,357 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:05:52,357 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:05:53,036 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 16:05:53,037 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 16:05:53,357 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:05:54,357 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:05:54,565 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:05:55,358 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:05:56,358 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:05:57,358 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:05:58,358 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:05:59,368 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:05:59,568 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:06:00,362 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:06:01,360 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:06:02,360 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:06:03,360 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:06:04,363 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:06:04,575 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:06:04,922 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 16:06:04,923 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 16:06:04,924 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 16:06:04,925 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 16:06:05,363 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:06:05,802 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 16:06:06,363 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:06:06,803 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 16:06:07,363 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:06:08,036 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 16:06:08,037 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 16:06:08,363 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:06:09,363 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:06:10,363 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:06:10,569 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:06:11,363 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:06:12,363 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:06:13,363 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:06:14,364 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:06:15,120 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 16:06:15,364 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:06:16,122 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:06:16,364 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:06:17,364 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:06:18,364 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:06:19,364 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:06:20,364 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:06:21,364 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:06:21,570 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:06:22,364 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:06:23,036 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 16:06:23,037 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 16:06:23,364 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:06:24,364 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:06:25,365 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:06:26,365 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:06:27,365 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:06:27,570 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:06:28,365 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:06:29,365 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:06:30,370 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:06:31,375 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:06:32,376 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:06:32,577 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:06:33,377 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:06:34,381 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:06:35,377 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:06:35,393 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 16:06:35,394 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 16:06:35,394 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 16:06:35,395 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 16:06:35,843 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 16:06:36,378 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:06:36,844 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 16:06:37,377 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:06:38,037 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 16:06:38,037 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 16:06:38,153 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:06:38,379 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:06:39,378 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:06:40,379 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:06:41,379 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:06:42,379 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:06:43,379 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:06:43,574 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:06:44,379 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:06:45,124 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 16:06:45,379 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:06:46,379 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:06:47,379 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:06:48,528 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:06:48,574 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:06:49,525 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:06:50,525 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:06:51,525 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:06:52,526 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:06:53,037 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 16:06:53,037 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 16:06:53,526 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:06:54,526 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:06:54,574 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:06:55,526 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:06:56,526 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:06:57,526 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:06:58,526 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:06:59,526 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:07:00,529 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:07:00,581 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:07:01,530 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:07:02,531 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:07:03,532 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:07:04,530 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:07:05,530 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:07:05,974 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 16:07:05,976 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 16:07:05,976 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 16:07:05,976 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:07:05,977 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 16:07:06,534 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:07:06,870 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 16:07:06,871 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 16:07:07,533 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:07:08,037 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 16:07:08,037 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 16:07:08,533 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:07:09,533 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:07:10,533 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:07:11,533 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:07:11,575 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:07:12,533 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:07:13,533 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:07:14,533 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:07:15,128 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 16:07:15,533 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:07:16,533 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:07:17,130 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:07:17,756 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:07:18,670 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:07:19,670 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:07:20,670 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:07:21,670 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:07:22,575 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:07:22,670 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:07:23,037 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 16:07:23,037 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 16:07:23,670 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:07:24,670 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:07:25,670 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:07:26,670 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:07:27,670 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:07:28,575 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:07:28,679 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:07:29,677 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:07:30,677 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:07:31,677 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:07:32,683 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:07:33,578 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:07:33,677 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:07:34,179 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 16:07:34,180 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 16:07:34,181 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 16:07:34,183 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 16:07:34,678 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:07:34,897 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 16:07:35,678 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:07:36,678 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:07:36,899 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 16:07:37,678 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:07:38,037 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 16:07:38,038 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 16:07:38,578 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:07:38,678 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:07:39,678 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:07:40,678 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:07:41,678 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:07:42,678 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:07:43,678 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:07:44,578 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:07:44,678 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:07:45,132 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 16:07:45,678 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:07:46,679 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:07:47,679 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:07:48,679 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:07:49,578 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:07:49,679 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:07:50,679 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:07:51,679 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:07:52,679 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:07:53,037 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 16:07:53,038 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 16:07:53,679 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:07:54,679 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:07:55,578 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:07:55,679 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:07:56,680 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:07:57,680 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:07:58,680 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:07:59,684 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:08:00,579 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:08:00,683 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:08:01,686 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:08:02,692 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:08:03,693 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:08:04,690 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:08:05,242 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 16:08:05,244 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 16:08:05,244 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 16:08:05,246 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 16:08:05,690 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:08:05,921 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 16:08:06,284 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:08:06,690 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:08:06,922 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 16:08:07,690 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:08:08,038 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 16:08:08,038 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 16:08:08,691 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:08:09,690 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:08:10,691 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:08:11,580 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:08:11,691 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:08:12,691 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:08:13,691 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:08:14,691 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:08:15,137 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 16:08:15,691 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:08:16,691 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:08:17,139 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:08:17,691 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:08:18,691 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:08:19,691 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:08:20,692 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:08:21,691 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:08:22,580 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:08:22,691 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:08:23,038 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 16:08:23,038 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 16:08:23,692 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:08:24,692 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:08:25,692 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:08:26,692 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:08:27,692 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:08:28,580 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:08:28,692 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:08:29,692 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:08:30,704 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:08:31,696 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:08:32,696 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:08:33,589 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:08:33,701 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:08:34,696 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:08:35,579 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 16:08:35,581 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 16:08:35,581 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 16:08:35,582 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 16:08:35,948 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 16:08:35,992 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:08:36,949 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 16:08:36,991 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:08:37,991 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:08:38,038 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 16:08:38,038 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 16:08:38,991 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:08:39,584 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:08:39,992 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:08:40,992 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:08:41,992 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:08:42,992 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:08:43,992 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:08:44,584 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:08:44,992 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:08:45,140 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 16:08:45,992 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:08:46,992 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:08:47,993 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:08:48,992 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:08:49,584 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:08:49,992 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:08:50,993 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:08:51,993 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:08:52,993 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:08:53,038 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 16:08:53,038 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 16:08:53,993 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:08:54,993 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:08:55,584 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:08:55,993 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:08:56,993 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:08:57,993 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:08:58,993 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:09:00,002 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:09:00,589 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:09:00,999 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:09:02,004 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:09:03,003 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:09:04,003 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:09:04,734 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 16:09:04,735 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 16:09:04,736 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 16:09:04,737 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 16:09:04,972 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 16:09:05,138 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:09:05,776 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:09:06,134 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:09:06,974 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 16:09:07,134 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:09:08,038 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 16:09:08,038 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 16:09:08,134 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:09:09,134 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:09:10,134 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:09:11,134 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:09:11,585 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:09:12,134 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:09:13,134 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:09:14,134 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:09:15,135 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:09:15,144 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 16:09:16,135 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:09:17,135 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:09:17,146 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:09:18,135 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:09:19,319 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:09:20,231 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:09:21,231 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:09:22,231 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:09:22,585 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:09:23,038 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 16:09:23,039 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 16:09:23,231 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:09:24,231 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:09:25,231 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:09:26,231 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:09:27,231 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:09:27,585 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:09:28,231 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:09:29,232 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:09:30,231 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:09:31,232 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:09:32,241 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:09:32,588 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:09:33,234 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:09:34,234 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:09:35,234 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:09:36,234 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:09:37,235 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:09:37,464 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 16:09:37,466 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 16:09:37,467 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 16:09:37,469 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 16:09:38,000 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 16:09:38,038 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 16:09:38,039 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 16:09:38,148 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:09:38,234 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:09:39,001 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 16:09:39,234 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:09:40,235 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:09:41,235 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:09:42,235 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:09:43,235 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:09:43,589 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:09:44,235 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:09:45,148 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 16:09:45,235 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:09:46,235 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:09:47,235 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:09:48,235 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:09:48,590 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:09:49,236 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:09:50,236 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:09:51,236 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:09:52,236 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:09:53,038 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 16:09:53,039 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 16:09:53,236 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:09:54,236 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:09:54,589 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:09:55,236 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:09:56,236 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:09:57,236 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:09:58,236 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:09:59,237 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:10:00,239 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:10:00,595 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:10:01,240 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:10:02,242 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:10:03,244 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:10:04,247 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:10:05,247 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:10:05,401 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 16:10:05,403 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 16:10:05,404 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 16:10:05,405 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 16:10:06,027 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 16:10:06,247 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:10:06,442 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:10:07,028 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 16:10:07,247 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:10:08,038 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 16:10:08,039 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 16:10:08,247 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:10:09,247 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:10:10,247 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:10:11,247 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:10:11,593 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:10:12,247 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:10:13,248 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:10:14,248 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:10:15,151 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 16:10:15,248 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:10:16,248 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:10:17,153 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:10:17,248 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:10:18,248 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:10:19,249 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:10:20,248 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:10:21,248 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:10:22,248 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:10:22,593 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:10:23,038 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 16:10:23,039 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 16:10:23,249 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:10:24,249 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:10:25,249 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:10:26,249 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:10:27,249 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:10:28,249 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:10:28,593 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:10:29,249 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:10:30,249 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:10:31,256 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:10:32,255 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:10:33,253 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:10:33,595 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:10:34,259 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:10:35,253 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:10:36,253 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:10:36,561 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 16:10:36,562 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 16:10:36,563 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 16:10:36,564 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 16:10:37,059 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 16:10:37,254 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:10:38,039 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 16:10:38,039 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 16:10:38,253 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:10:38,597 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:10:39,060 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 16:10:39,253 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:10:40,254 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:10:41,254 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:10:42,254 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:10:43,254 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:10:44,254 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:10:44,596 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:10:45,158 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 16:10:45,254 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:10:46,254 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:10:47,254 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:10:48,254 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:10:49,254 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:10:50,255 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:10:50,596 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:10:51,255 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:10:52,255 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:10:53,038 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 16:10:53,039 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 16:10:53,255 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:10:54,255 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:10:55,255 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:10:55,596 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:10:56,255 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:10:57,255 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:10:58,255 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:10:59,255 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:11:00,255 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:11:01,256 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:11:01,597 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:11:02,256 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:11:03,263 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:11:04,260 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:11:05,263 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:11:06,264 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:11:06,611 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:11:07,264 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:11:08,042 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 16:11:08,042 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 16:11:08,264 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:11:08,832 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 16:11:08,833 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 16:11:08,834 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 16:11:08,835 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 16:11:09,088 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 16:11:09,264 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:11:10,264 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:11:11,090 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 16:11:11,264 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:11:11,873 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:11:12,264 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:11:13,264 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:11:14,264 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:11:15,159 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 16:11:15,265 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:11:16,265 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:11:17,161 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:11:17,265 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:11:18,265 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:11:19,265 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:11:20,265 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:11:21,265 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:11:22,265 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:11:22,601 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:11:23,042 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 16:11:23,042 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 16:11:23,265 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:11:24,265 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:11:25,266 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:11:26,266 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:11:27,266 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:11:27,601 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:11:28,266 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:11:29,266 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:11:30,266 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:11:31,266 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:11:32,266 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:11:33,266 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:11:33,601 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:11:34,266 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:11:35,277 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:11:36,272 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:11:37,274 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:11:38,043 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 16:11:38,043 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 16:11:38,274 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:11:38,614 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:11:39,274 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:11:40,256 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 16:11:40,258 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 16:11:40,258 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 16:11:40,260 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 16:11:40,704 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:11:41,123 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 16:11:41,123 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 16:11:41,703 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:11:42,703 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:11:43,703 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:11:44,607 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:11:44,703 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:11:45,163 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 16:11:45,703 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:11:46,703 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:11:47,703 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:11:48,703 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:11:49,607 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:11:49,703 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:11:50,703 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:11:51,704 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:11:52,704 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:11:53,043 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 16:11:53,043 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 16:11:53,704 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:11:54,704 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:11:55,607 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:11:55,704 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:11:56,704 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:11:57,704 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:11:58,704 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:11:59,704 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:12:00,607 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:12:00,704 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:12:01,704 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:12:02,704 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:12:03,705 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:12:04,705 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:12:05,705 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:12:06,607 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:12:06,705 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:12:07,712 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:12:08,045 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 16:12:08,050 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 16:12:08,708 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:12:09,711 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:12:10,711 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:12:11,622 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:12:11,711 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:12:12,713 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:12:13,263 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 16:12:13,264 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 16:12:13,264 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 16:12:13,266 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 16:12:13,711 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:12:14,157 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 16:12:14,711 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:12:15,158 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 16:12:15,167 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 16:12:15,711 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:12:16,712 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:12:17,169 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:12:17,712 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:12:18,712 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:12:19,712 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:12:20,712 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:12:21,712 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:12:22,683 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:12:22,712 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:12:23,045 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 16:12:23,045 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 16:12:23,712 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:12:24,712 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:12:25,712 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:12:26,713 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:12:27,713 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:12:28,683 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:12:28,713 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:12:29,713 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:12:30,713 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:12:31,713 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:12:32,713 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:12:33,713 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:12:34,683 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:12:34,714 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:12:35,713 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:12:36,713 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:12:37,714 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:12:38,048 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 16:12:38,051 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 16:12:38,720 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:12:39,695 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:12:39,721 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:12:40,721 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:12:41,722 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:12:42,722 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:12:43,718 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 16:12:43,719 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 16:12:43,720 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 16:12:43,722 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 16:12:44,119 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:12:44,183 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 16:12:44,760 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:12:44,864 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:12:45,171 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 16:12:45,185 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 16:12:45,864 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:12:46,864 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:12:47,864 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:12:48,864 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:12:49,864 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:12:50,689 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:12:50,865 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:12:51,865 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:12:52,865 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:12:53,045 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 16:12:53,045 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 16:12:53,865 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:12:54,865 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:12:55,865 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:12:56,689 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:12:56,865 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:12:57,866 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:12:58,866 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:12:59,866 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:13:00,866 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:13:01,689 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:13:01,866 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:13:02,866 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:13:03,866 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:13:04,866 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:13:05,866 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:13:06,866 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:13:07,689 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:13:07,866 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:13:08,045 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 16:13:08,046 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 16:13:08,867 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:13:09,867 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:13:10,872 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:13:11,870 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:13:12,695 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:13:12,871 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:13:13,869 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:13:14,872 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:13:15,183 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 16:13:15,869 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:13:16,508 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 16:13:16,509 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 16:13:16,510 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 16:13:16,512 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 16:13:16,905 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:13:17,213 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 16:13:17,213 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 16:13:17,904 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:13:18,551 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:13:18,904 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:13:19,905 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:13:20,905 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:13:21,905 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:13:22,905 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:13:23,045 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 16:13:23,046 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 16:13:23,695 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:13:23,905 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:13:24,905 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:13:25,905 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:13:26,905 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:13:27,905 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:13:28,695 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:13:28,905 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:13:29,906 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:13:30,906 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:13:31,906 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:13:32,906 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:13:33,695 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:13:33,906 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:13:34,906 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:13:35,906 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:13:36,906 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:13:37,906 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:13:38,045 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 16:13:38,046 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 16:13:38,906 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:13:39,695 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:13:39,906 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:13:40,906 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:13:41,907 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:13:42,914 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:13:43,911 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:13:44,701 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:13:44,915 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:13:45,182 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 16:13:45,915 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:13:46,915 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:13:47,915 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:13:48,305 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 16:13:48,306 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 16:13:48,306 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 16:13:48,311 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 16:13:48,915 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:13:49,242 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 16:13:49,242 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 16:13:49,704 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:13:49,915 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:13:50,915 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:13:51,915 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:13:52,916 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:13:53,045 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 16:13:53,046 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 16:13:53,916 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:13:54,916 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:13:55,704 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:13:55,916 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:13:56,916 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:13:57,916 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:13:58,916 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:13:59,916 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:14:00,916 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:14:01,704 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:14:01,916 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:14:02,917 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:14:03,917 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:14:04,917 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:14:05,917 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:14:06,704 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:14:06,917 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:14:07,917 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:14:08,046 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 16:14:08,046 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 16:14:08,917 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:14:09,917 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:14:10,917 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:14:11,917 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:14:12,704 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:14:12,917 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:14:13,917 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:14:14,924 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:14:15,185 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 16:14:15,924 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:14:16,925 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:14:17,922 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:14:18,191 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:14:18,922 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:14:19,923 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:14:20,405 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 16:14:20,406 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 16:14:20,407 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 16:14:20,412 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 16:14:20,924 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:14:21,270 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 16:14:21,270 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 16:14:21,923 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:14:22,923 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:14:23,046 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 16:14:23,046 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 16:14:23,710 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:14:23,923 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:14:24,924 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:14:25,924 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:14:26,924 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:14:27,924 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:14:28,710 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:14:28,924 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:14:29,924 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:14:30,924 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:14:31,924 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:14:32,924 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:14:33,710 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:14:33,925 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:14:34,924 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:14:35,925 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:14:36,925 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:14:37,925 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:14:38,046 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 16:14:38,046 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 16:14:38,925 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:14:39,710 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:14:39,925 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:14:40,925 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:14:41,925 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:14:42,925 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:14:43,925 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:14:44,715 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:14:44,936 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:14:45,191 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 16:14:45,929 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:14:46,932 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:14:47,933 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:14:48,937 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:14:49,937 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:14:49,995 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 16:14:49,997 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 16:14:49,998 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 16:14:49,998 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:14:50,000 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 16:14:50,296 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 16:14:50,937 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:14:51,298 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 16:14:51,938 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:14:52,938 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:14:53,046 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 16:14:53,047 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 16:14:53,938 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:14:54,938 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:14:55,713 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:14:55,938 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:14:56,938 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:14:57,938 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:14:58,938 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:14:59,938 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:15:00,714 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:15:00,938 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:15:01,938 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:15:02,938 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:15:03,939 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:15:04,939 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:15:05,714 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:15:05,939 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:15:06,939 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:15:07,939 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:15:08,046 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 16:15:08,047 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 16:15:08,939 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:15:09,939 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:15:10,939 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:15:11,714 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:15:11,939 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:15:12,939 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:15:13,939 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:15:14,940 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:15:15,196 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 16:15:15,948 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:15:16,943 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:15:17,202 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:15:17,950 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:15:18,946 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:15:19,949 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:15:20,947 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:15:21,394 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 16:15:21,395 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 16:15:21,395 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 16:15:21,401 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 16:15:21,947 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:15:22,323 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 16:15:22,437 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:15:22,947 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:15:23,046 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 16:15:23,047 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 16:15:23,324 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 16:15:23,947 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:15:24,947 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:15:25,947 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:15:26,947 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:15:27,717 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:15:27,947 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:15:28,947 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:15:29,948 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:15:30,948 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:15:31,948 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:15:32,717 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:15:32,948 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:15:33,948 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:15:34,948 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:15:35,948 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:15:36,949 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:15:37,949 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:15:38,046 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 16:15:38,047 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 16:15:38,205 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:15:38,948 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:15:39,948 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:15:40,949 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:15:41,949 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:15:42,949 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:15:43,717 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:15:43,949 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:15:44,949 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:15:45,199 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 16:15:45,949 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:15:46,949 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:15:47,950 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:15:48,732 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:15:48,949 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:15:49,956 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:15:50,961 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:15:51,962 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:15:52,961 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:15:53,046 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 16:15:53,047 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 16:15:53,769 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 16:15:53,771 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 16:15:53,771 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 16:15:53,772 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:15:53,777 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 16:15:54,235 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:15:54,353 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 16:15:55,234 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:15:55,354 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 16:15:56,234 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:15:57,235 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:15:58,235 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:15:59,235 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:15:59,721 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:16:00,235 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:16:01,235 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:16:02,235 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:16:03,235 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:16:04,235 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:16:05,235 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:16:05,721 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:16:06,236 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:16:07,235 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:16:08,047 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 16:16:08,047 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 16:16:08,235 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:16:09,236 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:16:10,236 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:16:11,236 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:16:11,721 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:16:12,236 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:16:13,236 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:16:14,236 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:16:15,204 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 16:16:15,236 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:16:16,236 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:16:17,207 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:16:17,237 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:16:18,245 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:16:19,244 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:16:20,248 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:16:21,242 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:16:22,244 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:16:22,729 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:16:23,047 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 16:16:23,047 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 16:16:23,087 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 16:16:23,124 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 16:16:23,125 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 16:16:23,127 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 16:16:23,386 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 16:16:23,488 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:16:24,487 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:16:25,387 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 16:16:25,487 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:16:26,487 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:16:27,487 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:16:28,487 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:16:28,724 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:16:29,487 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:16:30,488 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:16:31,488 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:16:32,488 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:16:33,488 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:16:34,488 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:16:34,724 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:16:35,488 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:16:36,707 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:16:37,615 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:16:38,047 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 16:16:38,048 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 16:16:38,615 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:16:39,615 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:16:40,615 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:16:40,724 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:16:41,615 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:16:42,615 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:16:43,615 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:16:44,615 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:16:45,207 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 16:16:45,615 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:16:46,210 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:16:46,615 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:16:47,616 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:16:48,625 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:16:49,618 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:16:50,621 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:16:51,620 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:16:51,738 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:16:52,618 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:16:53,051 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 16:16:53,051 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 16:16:53,618 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:16:54,234 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 16:16:54,236 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 16:16:54,236 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 16:16:54,237 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 16:16:54,411 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 16:16:54,637 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:16:55,412 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 16:16:55,634 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:16:56,634 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:16:57,277 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:16:57,634 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:16:58,634 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:16:59,634 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:17:00,634 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:17:01,635 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:17:02,635 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:17:02,728 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:17:03,635 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:17:04,636 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:17:05,753 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:17:06,669 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:17:07,669 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:17:08,051 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 16:17:08,052 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 16:17:08,172 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:17:08,670 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:17:09,670 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:17:10,670 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:17:11,670 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:17:12,670 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:17:13,670 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:17:13,728 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:17:14,670 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:17:15,211 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 16:17:15,670 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:17:16,670 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:17:17,674 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:17:18,679 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:17:18,737 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:17:19,675 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:17:20,675 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:17:21,675 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:17:22,675 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:17:22,945 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 16:17:22,947 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 16:17:22,947 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 16:17:22,949 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 16:17:23,348 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 16:17:23,348 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 16:17:23,437 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 16:17:23,675 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:17:24,682 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:17:24,735 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:17:25,438 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 16:17:25,676 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:17:26,676 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:17:27,676 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:17:28,676 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:17:29,676 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:17:30,676 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:17:30,735 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:17:31,676 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:17:32,677 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:17:33,677 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:17:34,677 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:17:35,677 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:17:35,735 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:17:36,677 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:17:37,677 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:17:38,090 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 16:17:38,091 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 16:17:38,677 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:17:39,677 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:17:40,677 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:17:40,735 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:17:41,677 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:17:42,677 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:17:43,678 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:17:44,678 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:17:45,216 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 16:17:45,678 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:17:46,218 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:17:46,678 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:17:47,686 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:17:48,684 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:17:49,684 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:17:50,687 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:17:51,684 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:17:51,740 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:17:52,684 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:17:53,090 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 16:17:53,091 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 16:17:53,164 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 16:17:53,242 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 16:17:53,242 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 16:17:53,244 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 16:17:53,463 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 16:17:53,684 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:17:54,684 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:17:55,465 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 16:17:55,684 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:17:56,684 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:17:57,685 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:17:57,737 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:17:58,685 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:17:59,685 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:18:00,685 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:18:01,685 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:18:02,685 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:18:03,685 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:18:03,737 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:18:04,685 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:18:05,685 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:18:06,685 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:18:07,686 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:18:08,090 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 16:18:08,091 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 16:18:08,686 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:18:08,737 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:18:09,686 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:18:10,686 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:18:11,686 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:18:12,686 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:18:13,686 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:18:14,686 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:18:14,737 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:18:15,220 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 16:18:15,686 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:18:16,686 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:18:17,783 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:18:18,686 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:18:19,698 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:18:19,742 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:18:20,691 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:18:21,691 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:18:22,692 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:18:23,096 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 16:18:23,096 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 16:18:23,693 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:18:24,693 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:18:24,746 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:18:25,228 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 16:18:25,229 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 16:18:25,229 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 16:18:25,230 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 16:18:25,493 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 16:18:25,694 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:18:26,693 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:18:27,495 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 16:18:27,693 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:18:28,693 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:18:29,693 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:18:30,693 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:18:30,744 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:18:31,693 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:18:32,693 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:18:33,694 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:18:34,694 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:18:35,694 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:18:36,694 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:18:36,744 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:18:37,694 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:18:38,096 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 16:18:38,096 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 16:18:38,694 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:18:39,695 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:18:40,694 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:18:41,694 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:18:42,695 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:18:42,744 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:18:43,695 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:18:44,695 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:18:45,223 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 16:18:45,695 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:18:46,695 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:18:47,695 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:18:48,226 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:18:48,695 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:18:49,695 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:18:50,695 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:18:51,701 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:18:52,699 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:18:53,102 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 16:18:53,103 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 16:18:53,699 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:18:53,751 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:18:54,702 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:18:55,700 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:18:56,700 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:18:56,984 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 16:18:56,987 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 16:18:56,987 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 16:18:56,989 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 16:18:57,524 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 16:18:57,524 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 16:18:57,700 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:18:58,700 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:18:59,700 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:18:59,748 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:19:00,701 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:19:01,701 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:19:02,701 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:19:03,701 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:19:04,701 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:19:04,748 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:19:05,701 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:19:06,701 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:19:07,701 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:19:08,099 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 16:19:08,099 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 16:19:08,701 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:19:09,701 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:19:10,701 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:19:10,748 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:19:11,702 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:19:12,702 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:19:13,702 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:19:14,702 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:19:15,228 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 16:19:15,702 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:19:16,230 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:19:16,702 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:19:17,702 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:19:18,702 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:19:19,702 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:19:20,702 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:19:21,702 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:19:21,748 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:19:22,703 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:19:23,099 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 16:19:23,100 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 16:19:23,703 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:19:24,703 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:19:25,706 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:19:26,713 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:19:26,751 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:19:27,709 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:19:28,706 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:19:29,706 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:19:30,684 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 16:19:30,685 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 16:19:30,686 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 16:19:30,688 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 16:19:31,088 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:19:31,553 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 16:19:32,088 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:19:32,554 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 16:19:32,727 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:19:33,088 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:19:34,088 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:19:35,088 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:19:36,088 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:19:37,088 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:19:37,751 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:19:38,088 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:19:38,099 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 16:19:38,100 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 16:19:39,088 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:19:40,088 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:19:41,089 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:19:42,089 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:19:43,308 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:19:43,751 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:19:44,215 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:19:45,215 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:19:45,232 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 16:19:46,215 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:19:47,215 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:19:48,215 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:19:49,215 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:19:49,751 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:19:50,215 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:19:51,215 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:19:52,215 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:19:53,099 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 16:19:53,100 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 16:19:53,215 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:19:54,215 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:19:54,751 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:19:55,231 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:19:56,221 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:19:57,222 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:19:58,224 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:19:59,222 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:19:59,760 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:20:00,222 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:20:00,743 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 16:20:00,745 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 16:20:00,745 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 16:20:00,751 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 16:20:01,223 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:20:01,579 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 16:20:02,223 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:20:02,579 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 16:20:03,222 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:20:04,223 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:20:05,223 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:20:05,754 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:20:06,223 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:20:07,223 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:20:08,099 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 16:20:08,100 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 16:20:08,223 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:20:09,223 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:20:10,223 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:20:11,223 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:20:11,754 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:20:12,223 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:20:13,224 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:20:14,223 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:20:15,224 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:20:15,236 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 16:20:16,224 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:20:17,224 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:20:17,238 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:20:18,224 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:20:19,224 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:20:20,224 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:20:21,224 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:20:22,224 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:20:22,754 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:20:23,100 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 16:20:23,100 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 16:20:23,225 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:20:24,228 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:20:25,228 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:20:26,231 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:20:27,231 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:20:27,776 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:20:28,234 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:20:29,097 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 16:20:29,099 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 16:20:29,099 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 16:20:29,101 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 16:20:29,504 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:20:29,608 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 16:20:30,249 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:20:30,609 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 16:20:31,249 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:20:32,249 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:20:33,140 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:20:33,250 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:20:34,250 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:20:35,250 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:20:36,250 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:20:37,250 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:20:38,100 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 16:20:38,100 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 16:20:38,250 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:20:38,264 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:20:39,250 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:20:40,250 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:20:41,251 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:20:42,250 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:20:43,251 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:20:43,757 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:20:44,251 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:20:45,240 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 16:20:45,251 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:20:46,251 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:20:47,251 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:20:48,251 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:20:49,251 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:20:49,757 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:20:50,251 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:20:51,251 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:20:52,251 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:20:53,110 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 16:20:53,118 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 16:20:53,253 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:20:54,253 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:20:54,759 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:20:55,253 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:20:56,258 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:20:57,259 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:20:58,257 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:20:58,559 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 16:20:58,560 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 16:20:58,560 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 16:20:58,561 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 16:20:58,636 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 16:20:59,257 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:20:59,959 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:21:00,257 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:21:00,638 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 16:21:01,257 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:21:02,257 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:21:03,257 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:21:04,257 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:21:05,257 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:21:05,958 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:21:06,257 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:21:07,258 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:21:08,103 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 16:21:08,103 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 16:21:08,258 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:21:09,258 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:21:10,258 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:21:10,959 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:21:11,487 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:21:12,487 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:21:13,486 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:21:14,487 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:21:15,243 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 16:21:15,487 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:21:16,246 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:21:16,487 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:21:17,487 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:21:18,487 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:21:19,487 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:21:20,487 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:21:21,487 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:21:21,959 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:21:22,487 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:21:23,103 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 16:21:23,104 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 16:21:23,490 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:21:24,490 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:21:25,494 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:21:26,495 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:21:26,970 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:21:27,494 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:21:28,494 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:21:29,067 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 16:21:29,069 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 16:21:29,069 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 16:21:29,071 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 16:21:29,494 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:21:29,666 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 16:21:30,495 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:21:30,667 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 16:21:31,495 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:21:32,110 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:21:32,494 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:21:33,495 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:21:34,495 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:21:35,495 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:21:36,495 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:21:37,495 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:21:37,961 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:21:38,103 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 16:21:38,104 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 16:21:38,495 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:21:39,495 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:21:40,495 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:21:41,700 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:21:42,699 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:21:43,699 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:21:43,961 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:21:44,699 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:21:45,248 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 16:21:45,700 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:21:46,700 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:21:47,700 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:21:48,700 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:21:49,700 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:21:49,961 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:21:50,700 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:21:51,700 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:21:52,700 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:21:53,103 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 16:21:53,104 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 16:21:53,700 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:21:54,700 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:21:54,977 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:21:55,703 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:21:56,703 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:21:57,703 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:21:58,703 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:21:59,558 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 16:21:59,560 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 16:21:59,560 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 16:21:59,561 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 16:21:59,690 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 16:21:59,966 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:22:00,599 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:22:00,709 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:22:01,709 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:22:02,693 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 16:22:02,709 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:22:03,710 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:22:04,710 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:22:05,710 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:22:05,967 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:22:06,710 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:22:07,710 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:22:08,103 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 16:22:08,104 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 16:22:08,710 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:22:09,710 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:22:10,710 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:22:11,710 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:22:11,967 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:22:12,711 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:22:13,710 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:22:14,711 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:22:15,249 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 16:22:15,711 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:22:16,711 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:22:17,251 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:22:17,711 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:22:18,711 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:22:19,711 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:22:20,711 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:22:21,712 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:22:22,711 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:22:22,968 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:22:23,104 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 16:22:23,104 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 16:22:23,719 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:22:24,714 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:22:25,716 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:22:26,715 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:22:27,714 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:22:27,988 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:22:28,714 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:22:29,158 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 16:22:29,159 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 16:22:29,160 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 16:22:29,161 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 16:22:29,715 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:22:29,726 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 16:22:30,715 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:22:30,726 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 16:22:31,714 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:22:32,715 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:22:33,201 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:22:33,715 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:22:34,715 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:22:35,715 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:22:36,715 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:22:37,715 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:22:38,104 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 16:22:38,104 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 16:22:38,715 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:22:38,973 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:22:39,715 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:22:40,715 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:22:41,715 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:22:42,715 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:22:43,716 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:22:43,973 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:22:44,716 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:22:45,253 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 16:22:45,716 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:22:46,716 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:22:47,716 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:22:48,716 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:22:49,716 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:22:49,973 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:22:50,716 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:22:51,716 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:22:52,716 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:22:53,104 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 16:22:53,104 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 16:22:53,716 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:22:54,716 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:22:55,722 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:22:55,978 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:22:56,723 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:22:57,721 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:22:58,726 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:22:59,725 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:23:00,728 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:23:00,981 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:23:01,117 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 16:23:01,119 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 16:23:01,119 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 16:23:01,121 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 16:23:01,728 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:23:01,752 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 16:23:02,728 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:23:02,753 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 16:23:03,728 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:23:04,728 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:23:05,728 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:23:06,728 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:23:06,979 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:23:07,729 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:23:08,104 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 16:23:08,104 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 16:23:08,728 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:23:09,729 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:23:10,729 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:23:11,729 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:23:12,729 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:23:12,979 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:23:13,729 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:23:14,729 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:23:15,257 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 16:23:15,729 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:23:16,729 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:23:17,729 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:23:18,259 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:23:18,730 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:23:19,730 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:23:20,730 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:23:21,730 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:23:22,730 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:23:23,104 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 16:23:23,104 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 16:23:23,730 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:23:24,024 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:23:24,730 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:23:25,730 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:23:26,735 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:23:27,733 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:23:28,733 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:23:29,027 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:23:29,733 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:23:30,735 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:23:31,735 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:23:32,115 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 16:23:32,117 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 16:23:32,117 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 16:23:32,118 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 16:23:32,735 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:23:32,778 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 16:23:33,735 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:23:34,029 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:23:34,735 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:23:34,779 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 16:23:35,735 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:23:36,735 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:23:37,735 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:23:38,104 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 16:23:38,105 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 16:23:38,735 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:23:39,736 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:23:40,029 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:23:40,736 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:23:41,736 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:23:42,736 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:23:43,736 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:23:44,937 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:23:45,029 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:23:45,261 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 16:23:45,847 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:23:46,847 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:23:47,847 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:23:48,847 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:23:49,847 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:23:50,029 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:23:50,847 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:23:51,847 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:23:52,848 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:23:53,104 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 16:23:53,105 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 16:23:53,848 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:23:54,848 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:23:55,848 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:23:56,029 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:23:56,850 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:23:57,851 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:23:58,850 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:23:59,849 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:24:00,850 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:24:01,042 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:24:01,850 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:24:02,245 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 16:24:02,247 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 16:24:02,247 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 16:24:02,249 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 16:24:02,811 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 16:24:02,850 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:24:03,850 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:24:04,813 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 16:24:04,850 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:24:05,851 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:24:06,850 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:24:07,034 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:24:07,851 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:24:08,104 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 16:24:08,105 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 16:24:08,851 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:24:09,851 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:24:10,851 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:24:11,851 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:24:12,851 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:24:13,034 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:24:13,851 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:24:14,909 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:24:15,265 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 16:24:15,908 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:24:16,908 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:24:17,908 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:24:18,268 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:24:18,908 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:24:19,908 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:24:20,909 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:24:21,909 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:24:22,909 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:24:23,104 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 16:24:23,105 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 16:24:23,909 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:24:24,034 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:24:24,909 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:24:25,909 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:24:26,915 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:24:27,911 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:24:28,916 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:24:29,038 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:24:29,916 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:24:30,917 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:24:31,915 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:24:32,457 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 16:24:32,458 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 16:24:32,459 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 16:24:32,461 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 16:24:32,840 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 16:24:32,915 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:24:33,915 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:24:34,039 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:24:34,841 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 16:24:34,915 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:24:35,916 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:24:36,916 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:24:37,916 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:24:38,104 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 16:24:38,105 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 16:24:38,916 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:24:39,916 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:24:40,039 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:24:40,916 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:24:41,916 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:24:42,916 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:24:43,916 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:24:44,916 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:24:45,269 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 16:24:45,275 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:24:45,916 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:24:46,916 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:24:47,917 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:24:48,917 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:24:49,917 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:24:50,917 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:24:51,039 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:24:51,917 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:24:52,917 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:24:53,105 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 16:24:53,105 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 16:24:53,917 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:24:54,917 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:24:55,917 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:24:56,929 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:24:57,046 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:24:57,924 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:24:58,923 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:24:59,923 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:25:00,924 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:25:01,924 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:25:02,164 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 16:25:02,166 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 16:25:02,167 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 16:25:02,167 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:25:02,168 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 16:25:02,864 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 16:25:02,924 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:25:03,924 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:25:04,866 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 16:25:04,924 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:25:05,924 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:25:06,924 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:25:07,924 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:25:08,041 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:25:08,105 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 16:25:08,105 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 16:25:08,924 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:25:09,924 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:25:10,924 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:25:11,924 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:25:12,924 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:25:13,925 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:25:14,041 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:25:14,925 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:25:15,273 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 16:25:15,925 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:25:16,925 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:25:17,925 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:25:18,925 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:25:19,925 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:25:20,041 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:25:20,925 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:25:21,925 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:25:22,926 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:25:23,105 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 16:25:23,105 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 16:25:23,931 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:25:24,927 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:25:25,050 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:25:25,927 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:25:26,927 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:25:27,927 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:25:28,927 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:25:29,515 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 16:25:29,516 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 16:25:29,517 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 16:25:29,518 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 16:25:29,887 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 16:25:29,928 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:25:30,556 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:25:30,890 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 16:25:30,928 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:25:31,928 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:25:32,928 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:25:33,928 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:25:34,928 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:25:35,928 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:25:36,045 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:25:36,928 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:25:37,928 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:25:38,105 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 16:25:38,106 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 16:25:38,929 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:25:39,928 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:25:40,929 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:25:41,929 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:25:42,045 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:25:42,929 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:25:43,929 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:25:44,929 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:25:45,277 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 16:25:45,929 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:25:46,929 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:25:47,279 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:25:47,929 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:25:48,930 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:25:49,930 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:25:50,930 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:25:51,930 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:25:52,930 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:25:53,046 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:25:53,112 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 16:25:53,112 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 16:25:53,939 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:25:54,936 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:25:55,937 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:25:56,938 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:25:57,937 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:25:58,055 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:25:58,938 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:25:58,980 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 16:25:58,981 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 16:25:58,982 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 16:25:58,984 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 16:25:59,929 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 16:25:59,937 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:26:00,929 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 16:26:00,937 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:26:01,937 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:26:02,937 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:26:03,380 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:26:03,937 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:26:04,937 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:26:05,937 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:26:06,938 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:26:07,938 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:26:08,112 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 16:26:08,112 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 16:26:08,938 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:26:09,284 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:26:09,938 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:26:10,938 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:26:11,938 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:26:12,938 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:26:13,939 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:26:14,379 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:26:14,939 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:26:15,281 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 16:26:15,939 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:26:16,939 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:26:17,939 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:26:18,939 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:26:19,939 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:26:20,379 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:26:20,939 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:26:21,939 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:26:22,944 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:26:23,117 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 16:26:23,120 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 16:26:23,944 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:26:24,951 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:26:25,380 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:26:25,946 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:26:26,949 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:26:27,928 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 16:26:27,929 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 16:26:27,929 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 16:26:27,930 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 16:26:27,954 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 16:26:28,365 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:26:28,955 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 16:26:29,364 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:26:30,364 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:26:30,381 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:26:31,364 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:26:32,364 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:26:33,364 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:26:34,364 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:26:35,364 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:26:36,364 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:26:36,381 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:26:37,364 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:26:38,117 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 16:26:38,117 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 16:26:38,364 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:26:39,364 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:26:40,365 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:26:41,365 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:26:42,381 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:26:42,381 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:26:43,380 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:26:44,380 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:26:45,285 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 16:26:45,380 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:26:46,380 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:26:47,380 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:26:48,287 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:26:48,380 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:26:49,381 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:26:50,380 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:26:51,380 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:26:52,381 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:26:53,117 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 16:26:53,118 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 16:26:53,381 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:26:54,262 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:26:54,381 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:26:55,383 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:26:56,388 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:26:57,391 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:26:58,398 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:26:59,384 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:26:59,392 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:27:00,168 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 16:27:00,168 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 16:27:00,169 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 16:27:00,170 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 16:27:00,603 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:27:00,987 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 16:27:00,988 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 16:27:01,601 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:27:02,602 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:27:03,602 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:27:04,602 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:27:05,382 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:27:05,602 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:27:06,602 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:27:07,602 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:27:08,117 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 16:27:08,118 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 16:27:08,602 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:27:09,602 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:27:10,602 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:27:11,382 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:27:11,602 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:27:12,602 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:27:13,603 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:27:14,603 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:27:15,289 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 16:27:15,603 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:27:16,603 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:27:17,292 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:27:17,603 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:27:18,603 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:27:19,603 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:27:20,603 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:27:21,603 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:27:22,382 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:27:22,603 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:27:23,117 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 16:27:23,118 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 16:27:23,604 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:27:24,604 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:27:25,604 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:27:26,607 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:27:27,383 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:27:27,607 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:27:28,604 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:27:29,611 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:27:30,608 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:27:31,212 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 16:27:31,214 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 16:27:31,215 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 16:27:31,216 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 16:27:31,644 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:27:32,017 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 16:27:32,643 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:27:33,018 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 16:27:33,255 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:27:33,645 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:27:34,643 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:27:35,643 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:27:36,643 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:27:37,644 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:27:38,117 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 16:27:38,118 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 16:27:38,302 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:27:38,644 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:27:39,644 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:27:40,644 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:27:41,644 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:27:42,769 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:27:43,386 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:27:43,766 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:27:44,766 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:27:45,293 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 16:27:45,766 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:27:46,766 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:27:47,766 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:27:48,766 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:27:49,295 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:27:49,766 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:27:50,766 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:27:51,766 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:27:52,773 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:27:53,121 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 16:27:53,129 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 16:27:53,775 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:27:54,392 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:27:54,775 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:27:55,772 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:27:56,772 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:27:57,772 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:27:58,240 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 16:27:58,241 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 16:27:58,241 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 16:27:58,242 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 16:27:58,778 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:27:59,046 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 16:27:59,047 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 16:27:59,774 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:28:00,392 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:28:00,775 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:28:01,775 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:28:02,775 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:28:03,775 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:28:04,775 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:28:05,775 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:28:06,391 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:28:06,775 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:28:07,775 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:28:08,121 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 16:28:08,121 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 16:28:08,775 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:28:09,775 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:28:10,775 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:28:11,776 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:28:12,391 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:28:12,776 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:28:13,776 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:28:14,776 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:28:15,297 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 16:28:15,776 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:28:16,776 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:28:17,776 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:28:18,299 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:28:18,776 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:28:19,776 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:28:20,776 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:28:21,777 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:28:22,777 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:28:23,121 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 16:28:23,121 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 16:28:23,786 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:28:24,297 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:28:24,783 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:28:25,778 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:28:26,778 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:28:27,778 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:28:28,778 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:28:29,039 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 16:28:29,041 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 16:28:29,042 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 16:28:29,043 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 16:28:29,074 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 16:28:29,439 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:28:29,778 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:28:30,778 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:28:31,076 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 16:28:31,778 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:28:32,779 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:28:33,779 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:28:34,779 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:28:35,438 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:28:35,779 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:28:36,779 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:28:37,779 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:28:38,121 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 16:28:38,122 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 16:28:38,779 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:28:39,779 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:28:40,779 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:28:41,438 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:28:42,072 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:28:42,814 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:28:43,815 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:28:44,815 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:28:45,301 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 16:28:45,815 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:28:46,815 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:28:47,303 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:28:47,815 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:28:48,815 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:28:49,815 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:28:50,815 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:28:51,815 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:28:52,438 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:28:52,815 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:28:53,121 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 16:28:53,122 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 16:28:53,816 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:28:54,819 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:28:55,824 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:28:56,824 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:28:57,451 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:28:57,828 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:28:58,825 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:28:59,744 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 16:28:59,746 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 16:28:59,746 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 16:28:59,748 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 16:29:00,104 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 16:29:00,142 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:29:00,891 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:29:01,105 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 16:29:01,891 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:29:02,786 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:29:02,891 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:29:03,891 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:29:04,891 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:29:05,891 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:29:06,891 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:29:07,891 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:29:08,121 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 16:29:08,122 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 16:29:08,290 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:29:08,891 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:29:09,891 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:29:10,891 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:29:11,891 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:29:12,892 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:29:13,442 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:29:13,892 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:29:14,892 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:29:15,305 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 16:29:15,892 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:29:16,892 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:29:17,892 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:29:18,892 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:29:19,307 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:29:19,892 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:29:20,892 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:29:21,892 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:29:22,892 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:29:23,121 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 16:29:23,122 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 16:29:23,893 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:29:24,442 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:29:24,893 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:29:25,900 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:29:26,895 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:29:27,895 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:29:28,895 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:29:29,445 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:29:29,906 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:29:30,899 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:29:31,344 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 16:29:31,345 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 16:29:31,346 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 16:29:31,347 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 16:29:31,899 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:29:32,135 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 16:29:32,899 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:29:33,136 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 16:29:33,899 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:29:34,446 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:29:34,899 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:29:35,899 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:29:36,900 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:29:37,900 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:29:38,122 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 16:29:38,122 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 16:29:38,900 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:29:39,900 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:29:40,446 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:29:40,900 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:29:41,900 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:29:42,900 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:29:43,900 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:29:44,900 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:29:45,309 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 16:29:45,900 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:29:46,312 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:29:46,900 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:29:47,900 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:29:48,901 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:29:49,901 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:29:50,901 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:29:51,446 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:29:51,901 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:29:52,901 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:29:53,122 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 16:29:53,122 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 16:29:53,901 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:29:54,907 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:29:55,907 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:29:56,461 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:29:56,909 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:29:57,910 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:29:58,910 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:29:59,910 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:29:59,977 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 16:29:59,978 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 16:29:59,979 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 16:29:59,980 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 16:30:00,161 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 16:30:00,910 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:30:01,162 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 16:30:01,910 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:30:02,018 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:30:02,910 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:30:03,910 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:30:04,910 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:30:05,910 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:30:06,911 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:30:07,450 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:30:07,911 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:30:08,122 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 16:30:08,122 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 16:30:08,911 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:30:09,911 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:30:10,911 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:30:11,911 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:30:12,911 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:30:13,450 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:30:13,912 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:30:14,912 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:30:15,313 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 16:30:15,912 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:30:16,912 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:30:17,912 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:30:18,912 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:30:19,315 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:30:19,912 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:30:20,912 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:30:21,912 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:30:22,912 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:30:23,122 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 16:30:23,122 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 16:30:23,912 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:30:24,450 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:30:24,913 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:30:25,916 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:30:26,917 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:30:27,919 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:30:28,917 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:30:29,456 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:30:29,917 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:30:30,918 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:30:31,120 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 16:30:31,121 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 16:30:31,122 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 16:30:31,123 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 16:30:31,187 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 16:30:31,917 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:30:32,917 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:30:33,189 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 16:30:33,918 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:30:34,918 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:30:35,454 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:30:35,918 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:30:36,918 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:30:37,918 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:30:38,122 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 16:30:38,122 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 16:30:38,918 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:30:39,918 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:30:40,918 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:30:41,454 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:30:41,918 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:30:42,919 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:30:43,919 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:30:44,919 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:30:45,317 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 16:30:45,919 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:30:46,919 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:30:47,319 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:30:47,919 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:30:48,919 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:30:49,919 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:30:50,919 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:30:51,919 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:30:52,454 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:30:52,919 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:30:53,122 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 16:30:53,123 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 16:30:53,920 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:30:54,924 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:30:55,921 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:30:56,921 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:30:57,464 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:30:57,921 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:30:58,921 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:30:59,921 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:31:00,180 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 16:31:00,180 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 16:31:00,181 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 16:31:00,183 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 16:31:00,215 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 16:31:00,922 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:31:01,216 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 16:31:01,921 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:31:02,921 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:31:03,221 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:31:03,922 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:31:04,922 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:31:05,922 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:31:06,922 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:31:07,922 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:31:08,122 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 16:31:08,123 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 16:31:08,294 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:31:08,922 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:31:09,922 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:31:10,922 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:31:11,922 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:31:13,152 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:31:13,458 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:31:14,151 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:31:15,151 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:31:15,321 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 16:31:16,151 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:31:17,152 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:31:18,152 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:31:19,152 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:31:19,323 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:31:20,152 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:31:21,152 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:31:22,152 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:31:23,123 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 16:31:23,123 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 16:31:23,163 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:31:24,152 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:31:24,458 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:31:25,167 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:31:26,158 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:31:27,162 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:31:28,162 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:31:29,169 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:31:29,460 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:31:30,166 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:31:30,524 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 16:31:30,525 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 16:31:30,525 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 16:31:30,527 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 16:31:31,166 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:31:31,246 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 16:31:31,246 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 16:31:32,166 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:31:33,166 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:31:34,166 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:31:34,461 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:31:35,166 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:31:36,166 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:31:37,166 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:31:38,122 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 16:31:38,123 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 16:31:38,166 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:31:39,166 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:31:40,167 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:31:40,461 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:31:41,167 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:31:42,432 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:31:43,428 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:31:44,428 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:31:45,323 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 16:31:45,429 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:31:46,326 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:31:46,429 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:31:47,429 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:31:48,429 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:31:49,429 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:31:50,429 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:31:51,429 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:31:51,461 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:31:52,429 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:31:53,123 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 16:31:53,123 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 16:31:53,429 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:31:54,441 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:31:55,436 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:31:56,432 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:31:56,474 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:31:57,442 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:31:58,433 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:31:59,433 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:31:59,987 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 16:31:59,989 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 16:31:59,989 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 16:31:59,991 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 16:32:00,276 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 16:32:00,433 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:32:01,277 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 16:32:01,433 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:32:02,028 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:32:02,433 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:32:03,433 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:32:04,433 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:32:05,433 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:32:06,433 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:32:07,434 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:32:07,467 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:32:08,123 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 16:32:08,123 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 16:32:08,434 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:32:09,434 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:32:10,434 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:32:11,434 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:32:12,434 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:32:13,434 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:32:13,467 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:32:14,552 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:32:15,328 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 16:32:15,453 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:32:16,453 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:32:17,453 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:32:18,453 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:32:19,331 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:32:19,453 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:32:20,453 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:32:21,453 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:32:22,453 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:32:23,123 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 16:32:23,123 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 16:32:23,453 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:32:24,453 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:32:24,551 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:32:25,453 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:32:26,461 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:32:27,458 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:32:28,459 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:32:29,463 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:32:29,555 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:32:30,463 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:32:31,463 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:32:32,069 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 16:32:32,070 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 16:32:32,070 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 16:32:32,072 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 16:32:32,303 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 16:32:32,473 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:32:33,304 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 16:32:33,472 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:32:34,472 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:32:34,556 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:32:35,472 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:32:36,473 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:32:37,472 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:32:38,123 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 16:32:38,124 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 16:32:38,473 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:32:39,473 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:32:40,473 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:32:40,556 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:32:41,473 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:32:42,473 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:32:43,473 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:32:44,473 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:32:45,330 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 16:32:45,473 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:32:46,332 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:32:46,473 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:32:47,473 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:32:48,474 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:32:49,474 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:32:50,474 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:32:51,474 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:32:51,693 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:32:52,474 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:32:53,123 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 16:32:53,124 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 16:32:53,474 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:32:54,474 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:32:55,474 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:32:56,475 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:32:56,702 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:32:57,475 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:32:58,484 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:32:59,481 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:33:00,481 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:33:01,484 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:33:02,317 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 16:33:02,318 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 16:33:02,319 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 16:33:02,319 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:33:02,322 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 16:33:02,334 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 16:33:02,715 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:33:03,335 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 16:33:03,713 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:33:04,713 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:33:05,713 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:33:06,714 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:33:07,693 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:33:07,714 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:33:08,123 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 16:33:08,124 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 16:33:08,714 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:33:09,714 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:33:10,714 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:33:11,714 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:33:12,714 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:33:13,693 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:33:13,714 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:33:14,714 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:33:15,335 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 16:33:15,714 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:33:16,715 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:33:17,715 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:33:18,715 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:33:19,337 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:33:19,715 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:33:20,715 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:33:21,715 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:33:22,715 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:33:23,123 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 16:33:23,124 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 16:33:23,715 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:33:24,712 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:33:24,719 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:33:25,719 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:33:26,725 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:33:27,723 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:33:28,725 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:33:29,723 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:33:29,961 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 16:33:29,963 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 16:33:29,964 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 16:33:29,964 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:33:29,965 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 16:33:30,363 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 16:33:30,723 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:33:31,364 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 16:33:31,723 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:33:32,723 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:33:33,724 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:33:34,723 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:33:35,699 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:33:35,724 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:33:36,724 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:33:37,724 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:33:38,123 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 16:33:38,124 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 16:33:38,724 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:33:39,724 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:33:40,724 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:33:41,699 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:33:41,724 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:33:42,725 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:33:43,725 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:33:44,724 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:33:45,339 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 16:33:45,725 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:33:46,725 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:33:47,341 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:33:47,725 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:33:48,725 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:33:49,725 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:33:50,725 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:33:51,725 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:33:52,699 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:33:52,725 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:33:53,124 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 16:33:53,124 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 16:33:53,734 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:33:54,730 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:33:55,732 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:33:56,729 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:33:57,706 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:33:57,733 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:33:58,731 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:33:58,943 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 16:33:58,945 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 16:33:58,945 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 16:33:58,947 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 16:33:59,391 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 16:33:59,735 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:34:00,734 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:34:01,392 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 16:34:01,734 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:34:02,735 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:34:03,703 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:34:03,735 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:34:04,735 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:34:05,735 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:34:06,735 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:34:07,735 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:34:08,124 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 16:34:08,124 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 16:34:08,735 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:34:09,222 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:34:09,735 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:34:10,735 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:34:11,735 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:34:12,755 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:34:13,754 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:34:14,703 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:34:14,754 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:34:15,343 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 16:34:15,754 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:34:16,754 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:34:17,755 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:34:18,755 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:34:19,703 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:34:19,755 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:34:20,755 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:34:21,755 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:34:22,755 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:34:23,124 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 16:34:23,124 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 16:34:23,755 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:34:24,706 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:34:24,760 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:34:25,757 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:34:26,757 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:34:27,759 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:34:28,759 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:34:29,708 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:34:29,759 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:34:30,358 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 16:34:30,359 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 16:34:30,360 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 16:34:30,361 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 16:34:30,417 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 16:34:30,763 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:34:31,418 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 16:34:31,763 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:34:32,763 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:34:33,763 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:34:34,708 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:34:34,763 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:34:35,763 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:34:36,763 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:34:37,763 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:34:38,124 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 16:34:38,125 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 16:34:38,763 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:34:39,709 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:34:39,764 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:34:40,764 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:34:41,764 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:34:42,764 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:34:43,764 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:34:44,709 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:34:44,764 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:34:45,347 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 16:34:45,764 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:34:46,764 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:34:47,764 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:34:48,764 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:34:49,764 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:34:50,709 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:34:50,765 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:34:51,765 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:34:52,765 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:34:53,124 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 16:34:53,125 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 16:34:53,765 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:34:54,771 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:34:55,714 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:34:55,773 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:34:56,772 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:34:57,770 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:34:58,772 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:34:59,770 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:35:00,449 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 16:35:00,451 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 16:35:00,451 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 16:35:00,452 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 16:35:00,852 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:35:01,445 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 16:35:01,446 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 16:35:01,491 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:35:01,849 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:35:02,849 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:35:03,849 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:35:04,849 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:35:05,850 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:35:06,711 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:35:06,850 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:35:07,850 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:35:08,124 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 16:35:08,125 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 16:35:08,850 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:35:09,850 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:35:10,850 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:35:11,711 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:35:11,850 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:35:12,850 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:35:13,850 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:35:14,850 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:35:15,352 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 16:35:15,851 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:35:16,851 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:35:17,354 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:35:17,851 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:35:18,851 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:35:19,851 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:35:20,851 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:35:21,851 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:35:22,711 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:35:22,851 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:35:23,124 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 16:35:23,125 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 16:35:23,851 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:35:24,851 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:35:25,852 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:35:26,852 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:35:27,717 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:35:27,857 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:35:28,857 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:35:29,857 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:35:30,858 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:35:31,696 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 16:35:31,697 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 16:35:31,698 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 16:35:31,699 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 16:35:32,109 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:35:32,473 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 16:35:32,737 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:35:33,106 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:35:33,474 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 16:35:34,106 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:35:35,106 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:35:36,106 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:35:37,106 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:35:38,106 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:35:38,124 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 16:35:38,125 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 16:35:38,212 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:35:39,106 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:35:40,107 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:35:41,107 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:35:42,107 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:35:43,107 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:35:43,718 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:35:44,107 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:35:45,107 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:35:45,355 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 16:35:46,107 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:35:47,107 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:35:48,107 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:35:49,107 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:35:49,357 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:35:50,108 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:35:51,108 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:35:52,108 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:35:53,108 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:35:53,124 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 16:35:53,125 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 16:35:54,108 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:35:54,718 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:35:55,108 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:35:56,108 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:35:57,116 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:35:58,112 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:35:59,115 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:35:59,721 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:36:00,115 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:36:00,814 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 16:36:00,815 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 16:36:00,816 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 16:36:00,818 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 16:36:01,211 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:36:01,499 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 16:36:01,499 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 16:36:02,218 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:36:03,218 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:36:04,218 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:36:05,218 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:36:05,721 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:36:06,218 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:36:07,218 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:36:08,125 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 16:36:08,125 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 16:36:08,218 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:36:09,218 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:36:10,218 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:36:11,218 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:36:11,721 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:36:12,218 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:36:13,218 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:36:14,406 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:36:15,359 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 16:36:15,406 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:36:16,406 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:36:17,361 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:36:17,406 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:36:18,406 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:36:19,406 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:36:20,406 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:36:21,406 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:36:22,406 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:36:22,721 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:36:23,125 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 16:36:23,125 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 16:36:23,406 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:36:24,407 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:36:25,407 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:36:26,415 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:36:27,414 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:36:27,728 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:36:28,414 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:36:29,417 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:36:30,417 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:36:31,417 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:36:31,779 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 16:36:31,780 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 16:36:31,780 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 16:36:31,782 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 16:36:32,418 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:36:32,527 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 16:36:32,820 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:36:33,420 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:36:33,528 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 16:36:34,419 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:36:35,419 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:36:36,419 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:36:37,419 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:36:38,125 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 16:36:38,125 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 16:36:38,288 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:36:38,419 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:36:39,420 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:36:40,420 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:36:41,420 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:36:42,420 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:36:43,420 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:36:43,725 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:36:44,421 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:36:45,363 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 16:36:45,420 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:36:46,420 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:36:47,420 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:36:48,421 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:36:49,366 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:36:49,421 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:36:50,421 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:36:51,421 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:36:52,421 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:36:53,125 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 16:36:53,126 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 16:36:53,421 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:36:54,421 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:36:54,725 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:36:55,421 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:36:56,430 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:36:57,424 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:36:58,428 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:36:59,428 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:36:59,734 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:37:00,430 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:37:01,428 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:37:01,500 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 16:37:01,502 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 16:37:01,502 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 16:37:01,503 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 16:37:01,547 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 16:37:02,428 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:37:03,428 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:37:03,548 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 16:37:04,428 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:37:05,428 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:37:05,729 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:37:06,428 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:37:07,428 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:37:08,125 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 16:37:08,126 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 16:37:08,429 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:37:09,429 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:37:10,429 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:37:11,429 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:37:11,729 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:37:12,429 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:37:13,430 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:37:14,429 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:37:15,367 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 16:37:15,429 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:37:16,429 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:37:17,369 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:37:17,430 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:37:18,430 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:37:19,430 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:37:20,430 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:37:21,430 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:37:22,430 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:37:22,729 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:37:23,125 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 16:37:23,126 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 16:37:23,430 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:37:24,430 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:37:25,432 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:37:26,436 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:37:27,438 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:37:27,738 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:37:28,436 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:37:29,385 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 16:37:29,386 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 16:37:29,387 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 16:37:29,388 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 16:37:29,577 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 16:37:29,796 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:37:30,531 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:37:31,532 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:37:31,578 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 16:37:32,532 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:37:32,800 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:37:33,532 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:37:34,532 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:37:35,532 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:37:36,532 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:37:37,532 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:37:38,125 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 16:37:38,126 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 16:37:38,207 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:37:38,532 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:37:39,532 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:37:40,532 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:37:41,983 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:37:42,981 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:37:43,796 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:37:43,981 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:37:44,982 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:37:45,372 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 16:37:45,982 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:37:46,982 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:37:47,982 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:37:48,982 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:37:49,374 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:37:49,982 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:37:50,982 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:37:51,982 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:37:52,982 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:37:53,125 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 16:37:53,126 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 16:37:53,983 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:37:54,811 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:37:54,987 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:37:55,986 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:37:56,988 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:37:57,986 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:37:58,986 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:37:59,987 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:38:00,095 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 16:38:00,097 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 16:38:00,097 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 16:38:00,098 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:38:00,099 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 16:38:00,609 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 16:38:00,987 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:38:01,610 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 16:38:01,986 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:38:02,988 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:38:03,987 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:38:04,987 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:38:05,799 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:38:05,988 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:38:06,988 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:38:07,988 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:38:08,125 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 16:38:08,126 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 16:38:08,988 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:38:09,988 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:38:10,988 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:38:11,799 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:38:11,988 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:38:12,988 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:38:13,988 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:38:14,988 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:38:15,375 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 16:38:15,988 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:38:16,989 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:38:17,377 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:38:17,989 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:38:18,989 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:38:19,989 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:38:20,989 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:38:21,989 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:38:22,799 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:38:22,989 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:38:23,126 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 16:38:23,126 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 16:38:23,989 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:38:24,989 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:38:25,989 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:38:26,990 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:38:27,812 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:38:27,990 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:38:28,922 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 16:38:28,924 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 16:38:28,925 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 16:38:28,928 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 16:38:29,333 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:38:29,640 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 16:38:29,640 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 16:38:30,338 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:38:31,338 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:38:32,338 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:38:33,338 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:38:33,803 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:38:34,338 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:38:35,338 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:38:36,338 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:38:37,338 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:38:38,126 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 16:38:38,126 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 16:38:38,338 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:38:39,197 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:38:39,338 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:38:40,338 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:38:41,338 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:38:42,339 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:38:43,339 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:38:44,339 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:38:44,803 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:38:45,339 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:38:45,379 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 16:38:46,339 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:38:47,339 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:38:48,339 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:38:49,339 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:38:50,339 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:38:50,803 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:38:51,339 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:38:52,340 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:38:53,126 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 16:38:53,126 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 16:38:53,340 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:38:54,340 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:38:55,340 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:38:55,806 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:38:56,346 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:38:57,346 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:38:58,350 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:38:59,347 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:39:00,346 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:39:00,809 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:39:01,111 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 16:39:01,112 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 16:39:01,113 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 16:39:01,115 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 16:39:01,511 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:39:01,672 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 16:39:01,672 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 16:39:02,510 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:39:03,510 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:39:04,510 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:39:05,510 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:39:06,510 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:39:06,808 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:39:07,510 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:39:08,126 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 16:39:08,127 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 16:39:08,511 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:39:09,511 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:39:10,511 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:39:11,511 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:39:12,511 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:39:12,808 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:39:13,512 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:39:14,511 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:39:15,382 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 16:39:15,511 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:39:16,511 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:39:17,511 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:39:18,384 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:39:18,511 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:39:19,512 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:39:20,512 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:39:21,512 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:39:22,512 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:39:23,126 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 16:39:23,127 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 16:39:23,512 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:39:24,289 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:39:24,520 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:39:25,515 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:39:26,515 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:39:27,516 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:39:28,516 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:39:29,356 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 16:39:29,357 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 16:39:29,357 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 16:39:29,357 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:39:29,358 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 16:39:29,706 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 16:39:29,754 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:39:30,752 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:39:31,707 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 16:39:31,753 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:39:32,753 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:39:33,753 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:39:34,753 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:39:34,811 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:39:35,753 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:39:36,753 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:39:37,753 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:39:38,126 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 16:39:38,127 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 16:39:38,753 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:39:39,753 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:39:40,753 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:39:40,811 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:39:41,753 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:39:42,754 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:39:43,754 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:39:44,754 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:39:45,386 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 16:39:45,754 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:39:46,388 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:39:46,754 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:39:47,754 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:39:48,754 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:39:49,754 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:39:50,754 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:39:51,754 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:39:51,811 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:39:52,755 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:39:53,126 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 16:39:53,127 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 16:39:53,755 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:39:54,755 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:39:55,755 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:39:56,755 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:39:57,755 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:39:57,811 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:39:58,766 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:39:59,759 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:40:00,761 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:40:01,759 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:40:02,760 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:40:02,822 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:40:03,760 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:40:03,846 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 16:40:03,847 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 16:40:03,847 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 16:40:03,848 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 16:40:04,735 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 16:40:04,760 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:40:05,736 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 16:40:05,760 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:40:06,760 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:40:07,760 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:40:08,126 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 16:40:08,127 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 16:40:08,289 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:40:08,760 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:40:09,760 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:40:10,761 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:40:11,761 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:40:12,761 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:40:13,761 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:40:13,813 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:40:14,761 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:40:15,390 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 16:40:15,761 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:40:16,761 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:40:17,761 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:40:18,761 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:40:19,393 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:40:19,761 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:40:20,761 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:40:21,762 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:40:22,762 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:40:23,127 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 16:40:23,127 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 16:40:23,762 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:40:24,762 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:40:24,813 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:40:25,762 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:40:26,762 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:40:27,772 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:40:28,767 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:40:29,769 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:40:29,814 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:40:30,766 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:40:31,766 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:40:32,662 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 16:40:32,663 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 16:40:32,663 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 16:40:32,665 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 16:40:32,760 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 16:40:33,060 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:40:33,761 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 16:40:34,058 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:40:35,058 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:40:35,814 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:40:36,059 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:40:37,059 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:40:38,059 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:40:38,127 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 16:40:38,127 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 16:40:39,059 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:40:40,059 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:40:40,814 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:40:41,059 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:40:42,059 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:40:43,059 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:40:44,059 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:40:45,059 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:40:45,395 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 16:40:46,060 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:40:46,397 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:40:47,060 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:40:48,060 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:40:49,060 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:40:50,060 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:40:51,060 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:40:51,814 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:40:52,060 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:40:53,060 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:40:53,127 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 16:40:53,127 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 16:40:54,060 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:40:55,060 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:40:56,060 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:40:57,067 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:40:57,814 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:40:58,069 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:40:59,065 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:41:00,064 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:41:01,064 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:41:02,064 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:41:02,779 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 16:41:02,781 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 16:41:02,781 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 16:41:02,782 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 16:41:02,787 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 16:41:02,821 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:41:03,187 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:41:03,788 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 16:41:04,183 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:41:05,183 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:41:06,184 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:41:07,184 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:41:08,127 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 16:41:08,128 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 16:41:08,184 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:41:08,209 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:41:09,184 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:41:10,184 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:41:11,184 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:41:12,184 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:41:13,184 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:41:13,817 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:41:14,184 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:41:15,184 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:41:15,398 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 16:41:16,185 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:41:17,185 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:41:18,185 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:41:19,185 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:41:19,400 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:41:20,185 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:41:21,185 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:41:22,185 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:41:23,127 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 16:41:23,128 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 16:41:23,185 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:41:24,185 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:41:24,817 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:41:25,185 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:41:26,185 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:41:27,185 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:41:28,201 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:41:29,190 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:41:29,822 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:41:30,190 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:41:31,190 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:41:32,190 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:41:33,191 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:41:33,474 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 16:41:33,476 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 16:41:33,476 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 16:41:33,477 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 16:41:33,816 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 16:41:34,191 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:41:35,190 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:41:35,817 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 16:41:35,822 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:41:36,191 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:41:37,191 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:41:38,127 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 16:41:38,128 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 16:41:38,191 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:41:39,191 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:41:40,191 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:41:40,822 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:41:41,191 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:41:42,191 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:41:43,191 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:41:44,191 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:41:45,236 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:41:45,402 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 16:41:46,236 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:41:46,405 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:41:47,235 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:41:48,236 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:41:49,236 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:41:50,236 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:41:51,236 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:41:51,822 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:41:52,236 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:41:53,127 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 16:41:53,128 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 16:41:53,236 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:41:54,236 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:41:55,236 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:41:56,236 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:41:57,248 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:41:57,825 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:41:58,240 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:41:59,243 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:42:00,240 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:42:01,240 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:42:02,240 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:42:02,716 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 16:42:02,717 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 16:42:02,718 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 16:42:02,719 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 16:42:02,843 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 16:42:03,241 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:42:03,757 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:42:04,241 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:42:04,844 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 16:42:05,242 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:42:06,241 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:42:07,241 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:42:08,128 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 16:42:08,128 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 16:42:08,241 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:42:09,241 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:42:09,295 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:42:10,241 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:42:11,241 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:42:12,241 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:42:13,242 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:42:14,242 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:42:14,824 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:42:15,242 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:42:15,406 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 16:42:16,242 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:42:17,242 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:42:18,242 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:42:19,242 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:42:19,824 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:42:20,242 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:42:21,242 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:42:22,242 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:42:23,128 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 16:42:23,128 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 16:42:23,242 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:42:24,242 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:42:25,243 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:42:25,824 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:42:26,243 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:42:27,243 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:42:28,243 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:42:29,243 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:42:30,250 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:42:30,830 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:42:31,248 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:42:32,248 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:42:33,248 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:42:34,248 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:42:35,249 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:42:35,360 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 16:42:35,361 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 16:42:35,362 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 16:42:35,363 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 16:42:35,878 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 16:42:36,249 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:42:36,401 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:42:36,879 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 16:42:37,249 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:42:38,128 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 16:42:38,128 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 16:42:38,249 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:42:39,249 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:42:40,249 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:42:41,251 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:42:41,825 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:42:42,249 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:42:43,249 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:42:44,249 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:42:45,249 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:42:45,410 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 16:42:46,249 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:42:47,249 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:42:47,413 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:42:48,250 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:42:49,394 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:42:50,304 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:42:51,304 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:42:52,304 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:42:52,826 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:42:53,128 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 16:42:53,128 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 16:42:53,304 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:42:54,304 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:42:55,304 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:42:56,304 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:42:57,304 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:42:58,304 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:42:58,825 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:42:59,304 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:43:00,304 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:43:01,305 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:43:02,309 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:43:03,313 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:43:03,833 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:43:04,312 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:43:05,312 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:43:06,312 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:43:07,313 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:43:07,617 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 16:43:07,619 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 16:43:07,619 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 16:43:07,621 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 16:43:07,905 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 16:43:08,128 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 16:43:08,132 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 16:43:08,312 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:43:09,301 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:43:09,314 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:43:10,312 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:43:10,907 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 16:43:11,313 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:43:12,313 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:43:13,313 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:43:14,313 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:43:14,834 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:43:15,313 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:43:15,415 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 16:43:16,313 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:43:17,313 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:43:18,313 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:43:19,313 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:43:20,313 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:43:20,834 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:43:21,314 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:43:22,314 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:43:23,128 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 16:43:23,129 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 16:43:23,314 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:43:24,314 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:43:25,314 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:43:25,834 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:43:26,314 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:43:27,314 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:43:28,314 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:43:29,314 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:43:30,314 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:43:31,314 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:43:31,834 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:43:32,315 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:43:33,320 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:43:34,316 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:43:35,316 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:43:36,320 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:43:36,846 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:43:37,321 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:43:38,128 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 16:43:38,129 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 16:43:38,316 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:43:38,583 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 16:43:38,584 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 16:43:38,584 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 16:43:38,586 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 16:43:38,935 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 16:43:39,317 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:43:40,316 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:43:40,936 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 16:43:41,317 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:43:42,317 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:43:42,841 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:43:43,317 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:43:44,317 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:43:45,317 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:43:45,417 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 16:43:46,317 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:43:47,317 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:43:48,317 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:43:48,419 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:43:49,317 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:43:50,317 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:43:51,317 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:43:52,372 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:43:53,128 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 16:43:53,129 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 16:43:53,371 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:43:54,207 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:43:54,371 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:43:55,371 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:43:56,371 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:43:57,371 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:43:58,371 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:43:59,371 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:43:59,841 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:44:00,371 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:44:01,371 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:44:02,372 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:44:03,372 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:44:04,375 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:44:04,843 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:44:05,375 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:44:06,375 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:44:07,375 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:44:08,131 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 16:44:08,131 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 16:44:08,375 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:44:09,375 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:44:09,662 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 16:44:09,664 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 16:44:09,664 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 16:44:09,665 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 16:44:09,958 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 16:44:10,046 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:44:10,376 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:44:10,959 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 16:44:11,376 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:44:12,376 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:44:13,376 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:44:14,376 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:44:15,376 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:44:15,421 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 16:44:15,427 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:44:16,376 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:44:17,376 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:44:18,376 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:44:19,376 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:44:20,376 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:44:21,045 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:44:21,377 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:44:22,378 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:44:23,131 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 16:44:23,131 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 16:44:23,377 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:44:24,475 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:44:25,387 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:44:26,045 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:44:26,387 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:44:27,387 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:44:28,387 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:44:29,387 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:44:30,388 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:44:31,388 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:44:32,045 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:44:32,388 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:44:33,388 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:44:34,388 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:44:35,388 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:44:36,398 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:44:37,049 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:44:37,393 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:44:38,136 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 16:44:38,144 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 16:44:38,397 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:44:39,400 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:44:40,397 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:44:41,397 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:44:41,732 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 16:44:41,734 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 16:44:41,735 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 16:44:41,743 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 16:44:41,989 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 16:44:42,717 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:44:42,773 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:44:43,717 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:44:44,717 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:44:44,991 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 16:44:45,425 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 16:44:45,717 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:44:46,717 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:44:47,717 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:44:48,427 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:44:48,718 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:44:49,718 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:44:50,718 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:44:51,718 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:44:52,718 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:44:53,135 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 16:44:53,136 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 16:44:53,718 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:44:54,203 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:44:54,718 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:44:55,718 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:44:56,718 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:44:57,718 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:44:58,719 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:44:59,719 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:45:00,048 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:45:00,719 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:45:01,719 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:45:02,719 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:45:03,719 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:45:04,727 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:45:05,048 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:45:05,720 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:45:06,721 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:45:07,725 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:45:08,142 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 16:45:08,143 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 16:45:08,726 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:45:09,725 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:45:10,049 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:45:10,174 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 16:45:10,175 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 16:45:10,176 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 16:45:10,178 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 16:45:10,725 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:45:11,014 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 16:45:11,725 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:45:12,725 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:45:13,015 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 16:45:13,725 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:45:14,725 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:45:15,429 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 16:45:15,436 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:45:15,725 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:45:16,726 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:45:17,726 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:45:18,726 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:45:19,726 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:45:20,726 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:45:21,049 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:45:21,726 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:45:22,726 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:45:23,324 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 16:45:23,325 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 16:45:23,726 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:45:24,727 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:45:25,726 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:45:26,049 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:45:26,727 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:45:27,727 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:45:28,727 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:45:29,727 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:45:30,727 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:45:31,727 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:45:32,049 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:45:32,727 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:45:33,727 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:45:34,727 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:45:35,733 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:45:36,731 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:45:37,056 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:45:37,731 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:45:38,239 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 16:45:38,241 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 16:45:38,731 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:45:39,731 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:45:40,680 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 16:45:40,682 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 16:45:40,683 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 16:45:40,684 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 16:45:41,036 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 16:45:41,572 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:45:41,826 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:45:42,723 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:45:42,826 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:45:43,038 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 16:45:43,827 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:45:44,827 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:45:45,433 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 16:45:45,827 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:45:46,827 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:45:47,827 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:45:48,436 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:45:48,827 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:45:49,827 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:45:50,828 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:45:51,828 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:45:52,828 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:45:53,239 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 16:45:53,239 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 16:45:53,828 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:45:54,359 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:45:54,828 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:45:55,828 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:45:56,828 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:45:57,828 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:45:58,828 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:45:59,829 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:46:00,050 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:46:00,829 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:46:01,829 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:46:02,829 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:46:03,829 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:46:04,829 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:46:05,839 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:46:06,052 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:46:06,833 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:46:07,840 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:46:08,246 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 16:46:08,247 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 16:46:08,837 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:46:09,833 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:46:10,833 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:46:11,055 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:46:11,488 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 16:46:11,489 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 16:46:11,489 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 16:46:11,491 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 16:46:12,072 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 16:46:12,670 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:46:13,073 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 16:46:13,669 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:46:14,669 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:46:15,438 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 16:46:15,669 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:46:16,440 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:46:16,669 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:46:17,669 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:46:18,669 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:46:19,669 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:46:20,669 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:46:21,670 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:46:22,052 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:46:22,670 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:46:23,243 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 16:46:23,243 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 16:46:23,670 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:46:24,670 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:46:25,670 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:46:26,670 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:46:27,052 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:46:27,670 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:46:28,670 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:46:29,671 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:46:30,671 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:46:31,671 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:46:32,671 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:46:33,052 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:46:33,671 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:46:34,671 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:46:35,674 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:46:36,677 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:46:37,678 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:46:38,060 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:46:38,247 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 16:46:38,249 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 16:46:38,685 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:46:39,678 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:46:40,678 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:46:41,248 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 16:46:41,249 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 16:46:41,250 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 16:46:41,252 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 16:46:42,096 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 16:46:42,351 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:46:43,097 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 16:46:43,290 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:46:43,351 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:46:44,351 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:46:45,351 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:46:45,441 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 16:46:46,351 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:46:47,351 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:46:48,351 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:46:48,443 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:46:49,351 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:46:50,351 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:46:51,351 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:46:52,352 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:46:53,247 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 16:46:53,248 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 16:46:53,352 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:46:54,352 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:46:54,380 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:46:55,513 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:46:56,419 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:46:57,419 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:46:58,419 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:46:59,419 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:47:00,056 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:47:00,419 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:47:01,419 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:47:02,419 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:47:03,419 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:47:04,419 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:47:05,056 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:47:05,420 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:47:06,420 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:47:07,427 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:47:08,252 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 16:47:08,256 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 16:47:08,426 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:47:09,428 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:47:10,061 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:47:10,435 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:47:11,432 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:47:12,432 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:47:12,945 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 16:47:12,946 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 16:47:12,947 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 16:47:12,949 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 16:47:13,126 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 16:47:13,432 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:47:14,432 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:47:15,127 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 16:47:15,432 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:47:15,445 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 16:47:15,451 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:47:16,432 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:47:17,432 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:47:18,432 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:47:19,433 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:47:20,433 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:47:21,059 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:47:21,433 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:47:22,433 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:47:23,251 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 16:47:23,251 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 16:47:23,433 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:47:24,433 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:47:25,433 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:47:26,060 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:47:26,434 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:47:27,433 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:47:28,433 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:47:29,434 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:47:30,434 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:47:31,060 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:47:31,434 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:47:32,434 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:47:33,434 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:47:34,434 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:47:35,434 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:47:36,434 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:47:37,060 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:47:37,434 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:47:38,265 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 16:47:38,269 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 16:47:38,437 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:47:39,437 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:47:40,437 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:47:41,437 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:47:42,070 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:47:42,437 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:47:43,437 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:47:43,577 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 16:47:43,578 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 16:47:43,578 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 16:47:43,580 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 16:47:44,157 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 16:47:44,438 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:47:45,158 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 16:47:45,438 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:47:45,449 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 16:47:46,438 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:47:47,438 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:47:47,451 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:47:48,438 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:47:49,438 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:47:50,438 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:47:51,438 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:47:52,438 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:47:53,064 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:47:53,252 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 16:47:53,252 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 16:47:53,438 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:47:54,438 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:47:55,438 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:47:56,438 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:47:57,438 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:47:58,439 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:47:59,064 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:47:59,439 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:48:00,439 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:48:01,439 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:48:02,439 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:48:03,439 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:48:04,064 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:48:04,439 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:48:05,439 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:48:06,439 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:48:07,439 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:48:08,252 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 16:48:08,253 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 16:48:08,439 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:48:09,371 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:48:09,440 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:48:10,447 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:48:11,445 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:48:12,445 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:48:13,445 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:48:14,455 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:48:15,064 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:48:15,453 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:48:15,458 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 16:48:16,018 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 16:48:16,019 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 16:48:16,019 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 16:48:16,020 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 16:48:16,182 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 16:48:16,449 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:48:17,183 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 16:48:17,449 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:48:18,449 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:48:19,449 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:48:20,065 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:48:20,449 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:48:21,449 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:48:22,449 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:48:23,252 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 16:48:23,253 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 16:48:23,450 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:48:24,450 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:48:25,450 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:48:26,064 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:48:26,450 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:48:27,450 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:48:28,450 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:48:29,450 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:48:30,451 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:48:31,065 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:48:31,451 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:48:32,450 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:48:33,451 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:48:34,451 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:48:35,451 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:48:36,451 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:48:37,065 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:48:37,451 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:48:38,252 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 16:48:38,253 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 16:48:38,451 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:48:39,462 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:48:40,455 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:48:41,459 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:48:42,074 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:48:42,455 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:48:43,455 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:48:44,323 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 16:48:44,324 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 16:48:44,325 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 16:48:44,326 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 16:48:44,758 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:48:45,209 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 16:48:45,455 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 16:48:45,473 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:48:46,473 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:48:47,211 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 16:48:47,457 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:48:47,473 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:48:48,473 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:48:49,473 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:48:50,473 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:48:51,473 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:48:52,473 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:48:53,069 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:48:53,252 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 16:48:53,253 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 16:48:53,473 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:48:54,473 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:48:55,473 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:48:56,474 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:48:57,474 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:48:58,474 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:48:59,069 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:48:59,474 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:49:00,474 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:49:01,474 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:49:02,474 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:49:03,474 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:49:04,069 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:49:04,474 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:49:05,474 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:49:06,475 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:49:07,475 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:49:08,253 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 16:49:08,253 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 16:49:08,475 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:49:09,414 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:49:09,478 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:49:10,476 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:49:11,479 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:49:12,476 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:49:13,476 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:49:14,476 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:49:14,761 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 16:49:14,763 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 16:49:14,763 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 16:49:14,763 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:49:14,765 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 16:49:15,235 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 16:49:15,458 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 16:49:15,477 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:49:16,477 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:49:17,236 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 16:49:17,477 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:49:18,477 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:49:19,477 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:49:20,175 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:49:20,477 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:49:21,477 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:49:22,477 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:49:23,253 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 16:49:23,253 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 16:49:23,477 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:49:24,477 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:49:25,477 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:49:26,175 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:49:26,478 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:49:27,478 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:49:28,478 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:49:29,478 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:49:30,478 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:49:31,176 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:49:31,478 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:49:32,478 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:49:33,478 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:49:34,478 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:49:35,478 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:49:36,478 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:49:37,175 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:49:37,479 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:49:38,253 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 16:49:38,253 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 16:49:38,479 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:49:39,482 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:49:40,483 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:49:41,484 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:49:42,198 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:49:42,484 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:49:43,487 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:49:44,485 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:49:44,699 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 16:49:44,701 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 16:49:44,701 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 16:49:44,702 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 16:49:45,270 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 16:49:45,463 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 16:49:45,485 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:49:46,485 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:49:47,271 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 16:49:47,465 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:49:47,485 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:49:48,485 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:49:49,485 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:49:50,485 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:49:51,485 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:49:52,486 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:49:53,180 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:49:53,253 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 16:49:53,253 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 16:49:53,486 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:49:54,486 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:49:55,486 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:49:56,486 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:49:57,487 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:49:58,180 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:49:58,487 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:49:59,487 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:50:00,487 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:50:01,487 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:50:02,487 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:50:03,180 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:50:03,487 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:50:04,487 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:50:05,487 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:50:06,487 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:50:07,488 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:50:08,184 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:50:08,262 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 16:50:08,265 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 16:50:08,488 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:50:09,491 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:50:10,489 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:50:11,488 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:50:12,490 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:50:13,187 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:50:13,490 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:50:13,856 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 16:50:13,858 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 16:50:13,858 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 16:50:13,860 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 16:50:14,295 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 16:50:14,490 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:50:15,296 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 16:50:15,467 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 16:50:15,490 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:50:16,490 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:50:17,490 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:50:18,469 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:50:18,490 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:50:19,490 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:50:20,491 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:50:21,491 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:50:22,491 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:50:23,259 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 16:50:23,259 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 16:50:23,491 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:50:24,428 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:50:24,491 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:50:25,491 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:50:26,491 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:50:27,491 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:50:28,515 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:50:29,514 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:50:30,182 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:50:30,514 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:50:31,514 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:50:32,514 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:50:33,514 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:50:34,514 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:50:35,515 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:50:36,182 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:50:36,515 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:50:37,515 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:50:38,259 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 16:50:38,259 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 16:50:38,515 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:50:39,515 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:50:40,515 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:50:41,515 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:50:42,182 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:50:42,526 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:50:43,521 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:50:44,519 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:50:45,471 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 16:50:45,521 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:50:46,525 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:50:47,472 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:50:47,519 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:50:48,212 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 16:50:48,213 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 16:50:48,213 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 16:50:48,215 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 16:50:48,324 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 16:50:48,589 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:50:49,325 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 16:50:49,587 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:50:50,587 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:50:51,587 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:50:52,587 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:50:53,185 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:50:53,259 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 16:50:53,260 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 16:50:53,588 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:50:54,588 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:50:55,588 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:50:56,588 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:50:57,588 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:50:58,588 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:50:59,185 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:50:59,588 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:51:00,588 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:51:01,589 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:51:02,588 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:51:03,589 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:51:04,185 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:51:04,589 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:51:05,589 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:51:06,589 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:51:07,589 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:51:08,259 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 16:51:08,260 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 16:51:08,589 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:51:09,397 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:51:09,589 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:51:10,589 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:51:11,589 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:51:12,589 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:51:13,593 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:51:14,593 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:51:15,196 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:51:15,476 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 16:51:15,596 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:51:16,597 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:51:17,600 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:51:18,597 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:51:19,253 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 16:51:19,254 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 16:51:19,255 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 16:51:19,256 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 16:51:19,352 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 16:51:19,655 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:51:20,651 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:51:21,187 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:51:21,353 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 16:51:21,651 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:51:22,651 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:51:23,259 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 16:51:23,260 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 16:51:23,651 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:51:24,651 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:51:25,651 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:51:26,651 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:51:27,187 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:51:27,652 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:51:28,652 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:51:29,652 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:51:30,652 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:51:31,652 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:51:32,187 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:51:32,652 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:51:33,653 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:51:34,652 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:51:35,652 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:51:36,653 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:51:37,188 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:51:37,653 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:51:38,259 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 16:51:38,260 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 16:51:38,653 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:51:39,653 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:51:40,653 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:51:41,653 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:51:42,653 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:51:43,188 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:51:43,653 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:51:44,653 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:51:45,477 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 16:51:45,654 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:51:46,661 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:51:47,654 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:51:48,484 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:51:48,654 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:51:49,657 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:51:50,654 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:51:51,635 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 16:51:51,636 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 16:51:51,637 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 16:51:51,642 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 16:51:52,054 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:51:52,383 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 16:51:52,785 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:51:53,260 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 16:51:53,260 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 16:51:53,384 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 16:51:53,785 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:51:54,397 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:51:54,786 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:51:55,786 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:51:56,786 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:51:57,786 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:51:58,786 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:51:59,787 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:52:00,190 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:52:00,786 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:52:01,786 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:52:02,786 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:52:03,786 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:52:04,787 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:52:05,190 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:52:05,787 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:52:06,787 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:52:07,787 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:52:08,260 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 16:52:08,260 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 16:52:08,787 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:52:09,787 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:52:10,787 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:52:11,190 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:52:11,787 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:52:12,788 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:52:13,788 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:52:14,788 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:52:15,481 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 16:52:15,788 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:52:16,492 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:52:16,792 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:52:17,799 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:52:18,800 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:52:19,803 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:52:20,801 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:52:21,801 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:52:21,810 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 16:52:21,811 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 16:52:21,812 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 16:52:21,812 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:52:21,814 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 16:52:22,409 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 16:52:22,801 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:52:23,260 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 16:52:23,260 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 16:52:23,411 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 16:52:23,801 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:52:24,801 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:52:25,801 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:52:26,801 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:52:27,193 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:52:27,802 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:52:28,802 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:52:29,802 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:52:30,802 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:52:31,802 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:52:32,802 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:52:33,193 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:52:33,802 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:52:34,802 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:52:35,802 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:52:36,802 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:52:37,803 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:52:38,247 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:52:38,260 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 16:52:38,260 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 16:52:38,803 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:52:39,803 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:52:40,803 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:52:41,803 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:52:42,803 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:52:43,803 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:52:44,247 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:52:44,803 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:52:45,485 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 16:52:45,803 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:52:46,804 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:52:47,814 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:52:48,810 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:52:49,496 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:52:49,810 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:52:50,813 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:52:51,813 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:52:52,625 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 16:52:52,627 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 16:52:52,627 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 16:52:52,632 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 16:52:53,034 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:52:53,260 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 16:52:53,261 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 16:52:53,439 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 16:52:54,034 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:52:55,034 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:52:55,253 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:52:55,441 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 16:52:56,034 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:52:57,034 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:52:58,034 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:52:59,034 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:53:00,034 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:53:01,034 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:53:01,251 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:53:02,034 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:53:03,035 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:53:04,035 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:53:05,035 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:53:06,035 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:53:06,251 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:53:07,035 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:53:08,035 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:53:08,260 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 16:53:08,261 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 16:53:09,035 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:53:10,035 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:53:11,035 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:53:11,251 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:53:12,035 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:53:13,035 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:53:14,036 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:53:15,036 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:53:15,489 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 16:53:16,036 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:53:16,491 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:53:17,041 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:53:18,038 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:53:19,038 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:53:20,038 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:53:21,038 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:53:22,038 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:53:22,259 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:53:22,579 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 16:53:22,581 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 16:53:22,582 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 16:53:22,583 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 16:53:23,039 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:53:23,260 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 16:53:23,261 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 16:53:23,477 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 16:53:24,038 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:53:25,038 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:53:25,479 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 16:53:26,039 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:53:27,039 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:53:28,039 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:53:28,257 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:53:29,039 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:53:30,039 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:53:31,039 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:53:32,039 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:53:33,039 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:53:34,039 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:53:34,257 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:53:35,039 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:53:36,040 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:53:37,040 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:53:38,040 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:53:38,260 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 16:53:38,261 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 16:53:39,040 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:53:39,337 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:53:40,040 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:53:41,040 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:53:42,040 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:53:43,040 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:53:44,040 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:53:45,040 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:53:45,257 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:53:45,493 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 16:53:46,041 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:53:47,041 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:53:48,041 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:53:49,041 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:53:50,049 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:53:50,262 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:53:51,047 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:53:52,047 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:53:53,049 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:53:53,271 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 16:53:53,272 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 16:53:54,049 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:53:55,019 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 16:53:55,019 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 16:53:55,020 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 16:53:55,026 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 16:53:55,423 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:53:55,427 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:53:55,519 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 16:53:56,170 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:53:57,173 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:53:57,521 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 16:53:58,172 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:53:59,173 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:54:00,173 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:54:01,173 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:54:01,423 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:54:02,173 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:54:03,173 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:54:04,173 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:54:05,173 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:54:06,173 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:54:07,351 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:54:07,423 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:54:08,263 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 16:54:08,263 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 16:54:08,350 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:54:09,350 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:54:10,350 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:54:11,351 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:54:12,351 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:54:13,351 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:54:13,423 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:54:14,352 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:54:15,351 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:54:15,498 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 16:54:16,351 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:54:17,355 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:54:18,356 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:54:18,507 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:54:19,355 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:54:20,358 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:54:21,357 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:54:22,026 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 16:54:22,027 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 16:54:22,028 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 16:54:22,029 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 16:54:22,426 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:54:22,544 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 16:54:23,263 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 16:54:23,264 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 16:54:23,426 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:54:23,545 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 16:54:24,340 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:54:24,426 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:54:25,426 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:54:26,426 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:54:27,426 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:54:28,426 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:54:29,426 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:54:29,429 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:54:30,426 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:54:31,426 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:54:32,426 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:54:33,426 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:54:34,427 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:54:34,429 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:54:35,427 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:54:36,427 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:54:37,427 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:54:38,263 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 16:54:38,264 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 16:54:38,427 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:54:39,427 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:54:40,383 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:54:40,427 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:54:41,427 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:54:42,427 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:54:43,428 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:54:44,428 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:54:45,428 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:54:45,429 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:54:45,501 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 16:54:46,428 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:54:47,432 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:54:48,438 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:54:49,439 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:54:50,440 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:54:50,443 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:54:51,437 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:54:52,147 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 16:54:52,148 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 16:54:52,149 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 16:54:52,150 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 16:54:52,553 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:54:52,574 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 16:54:53,263 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 16:54:53,264 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 16:54:53,552 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:54:53,576 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 16:54:54,553 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:54:55,553 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:54:56,434 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:54:56,553 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:54:57,553 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:54:58,553 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:54:59,554 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:55:00,553 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:55:01,553 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:55:02,434 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:55:02,553 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:55:03,553 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:55:04,554 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:55:05,554 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:55:06,554 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:55:07,434 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:55:07,554 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:55:08,263 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 16:55:08,264 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 16:55:08,554 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:55:09,554 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:55:10,554 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:55:11,554 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:55:12,434 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:55:12,554 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:55:13,554 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:55:14,555 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:55:15,504 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 16:55:15,555 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:55:16,555 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:55:17,507 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:55:17,555 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:55:18,563 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:55:19,559 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:55:20,564 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:55:21,564 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:55:22,566 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:55:23,267 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 16:55:23,267 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 16:55:23,332 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:55:23,565 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:55:23,872 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 16:55:23,874 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 16:55:23,875 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 16:55:23,876 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 16:55:24,565 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:55:24,608 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 16:55:25,565 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:55:25,610 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 16:55:26,565 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:55:27,565 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:55:28,437 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:55:28,565 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:55:29,565 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:55:30,565 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:55:31,565 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:55:32,566 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:55:33,566 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:55:34,437 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:55:34,566 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:55:35,566 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:55:36,566 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:55:37,613 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:55:38,267 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 16:55:38,268 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 16:55:38,612 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:55:39,612 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:55:40,421 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:55:40,612 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:55:41,612 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:55:42,613 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:55:43,613 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:55:44,613 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:55:45,437 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:55:45,507 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 16:55:45,613 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:55:46,613 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:55:47,613 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:55:48,613 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:55:49,620 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:55:50,441 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:55:50,618 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:55:51,619 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:55:52,618 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:55:53,273 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 16:55:53,274 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 16:55:53,618 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:55:54,618 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:55:55,006 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 16:55:55,007 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 16:55:55,007 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 16:55:55,009 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 16:55:55,439 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:55:55,618 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:55:55,642 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 16:55:55,643 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 16:55:56,618 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:55:57,618 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:55:58,619 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:55:59,619 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:56:00,619 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:56:01,439 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:56:01,619 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:56:02,619 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:56:03,619 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:56:04,619 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:56:05,619 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:56:06,686 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:56:07,439 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:56:07,685 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:56:08,270 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 16:56:08,270 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 16:56:08,685 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:56:09,685 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:56:10,685 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:56:11,685 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:56:12,685 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:56:13,439 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:56:13,685 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:56:14,686 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:56:15,511 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 16:56:15,686 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:56:16,686 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:56:17,686 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:56:18,521 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:56:18,692 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:56:19,691 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:56:20,693 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:56:21,691 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:56:22,692 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:56:23,274 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 16:56:23,274 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 16:56:23,691 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:56:23,994 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 16:56:23,996 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 16:56:23,996 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 16:56:23,997 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:56:23,998 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 16:56:24,668 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 16:56:24,692 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:56:25,669 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 16:56:25,696 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:56:26,696 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:56:27,696 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:56:28,696 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:56:29,446 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:56:29,696 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:56:30,696 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:56:31,696 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:56:32,696 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:56:33,696 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:56:34,696 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:56:35,446 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:56:35,697 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:56:36,697 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:56:37,697 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:56:38,274 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 16:56:38,274 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 16:56:38,812 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:56:39,726 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:56:40,446 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:56:40,726 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:56:41,726 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:56:42,726 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:56:43,726 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:56:44,726 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:56:45,516 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 16:56:45,522 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:56:45,727 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:56:46,727 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:56:47,727 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:56:48,727 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:56:49,727 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:56:50,727 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:56:51,446 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:56:51,736 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:56:52,736 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:56:53,278 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 16:56:53,279 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 16:56:53,736 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:56:54,736 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:56:55,736 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:56:56,452 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:56:56,739 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:56:57,223 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 16:56:57,225 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 16:56:57,225 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 16:56:57,227 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 16:56:57,699 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 16:56:57,700 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 16:56:57,737 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:56:58,736 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:56:59,736 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:57:00,736 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:57:01,737 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:57:02,452 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:57:02,737 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:57:03,737 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:57:04,737 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:57:05,737 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:57:06,737 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:57:07,737 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:57:08,277 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 16:57:08,277 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 16:57:08,430 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:57:08,737 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:57:09,738 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:57:10,737 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:57:11,738 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:57:12,738 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:57:13,456 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:57:13,738 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:57:14,738 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:57:15,519 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 16:57:15,738 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:57:16,738 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:57:17,738 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:57:18,522 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:57:18,738 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:57:19,738 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:57:20,739 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:57:21,745 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:57:22,748 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:57:23,283 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 16:57:23,284 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 16:57:23,746 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:57:24,426 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:57:24,746 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:57:25,746 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:57:26,618 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 16:57:26,619 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 16:57:26,619 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 16:57:26,620 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 16:57:26,726 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 16:57:27,022 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:57:27,727 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 16:57:28,018 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:57:29,019 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:57:29,661 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:57:30,019 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:57:31,019 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:57:32,019 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:57:33,019 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:57:34,019 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:57:35,019 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:57:35,459 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:57:36,019 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:57:37,019 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:57:38,019 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:57:38,281 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 16:57:38,281 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 16:57:39,019 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:57:40,020 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:57:40,459 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:57:41,020 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:57:42,020 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:57:43,020 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:57:44,020 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:57:45,020 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:57:45,523 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 16:57:45,530 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:57:46,020 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:57:47,020 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:57:48,020 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:57:49,020 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:57:50,020 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:57:51,021 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:57:51,459 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:57:52,021 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:57:53,021 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:57:53,288 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 16:57:53,289 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 16:57:54,024 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:57:55,026 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:57:56,033 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:57:56,462 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:57:57,030 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:57:58,033 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:57:59,009 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 16:57:59,011 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 16:57:59,011 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 16:57:59,013 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 16:57:59,414 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:57:59,756 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 16:57:59,756 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 16:58:00,412 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:58:01,412 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:58:02,413 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:58:02,461 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:58:03,413 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:58:04,413 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:58:05,413 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:58:06,413 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:58:07,413 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:58:08,285 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 16:58:08,285 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 16:58:08,413 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:58:08,422 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:58:09,413 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:58:10,413 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:58:11,415 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:58:12,413 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:58:13,414 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:58:13,461 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:58:14,414 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:58:15,414 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:58:15,527 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 16:58:16,414 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:58:17,414 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:58:18,414 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:58:18,530 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:58:19,414 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:58:20,414 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:58:21,414 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:58:22,414 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:58:23,285 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 16:58:23,285 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 16:58:23,414 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:58:24,415 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:58:24,455 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:58:25,415 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:58:26,426 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:58:27,418 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:58:28,418 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:58:29,418 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:58:29,471 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:58:30,418 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:58:31,418 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:58:31,642 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 16:58:31,643 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 16:58:31,644 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 16:58:31,646 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 16:58:31,781 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 16:58:32,419 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:58:33,418 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:58:33,782 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 16:58:34,418 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:58:34,685 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:58:35,419 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:58:36,419 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:58:37,419 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:58:38,285 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 16:58:38,285 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 16:58:38,420 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:58:39,419 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:58:40,419 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:58:40,421 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:58:41,419 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:58:42,419 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:58:43,420 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:58:44,419 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:58:45,419 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:58:45,465 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:58:45,531 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 16:58:46,420 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:58:47,420 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:58:48,420 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:58:49,420 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:58:50,420 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:58:50,465 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:58:51,421 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:58:52,420 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:58:53,285 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 16:58:53,286 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 16:58:53,420 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:58:54,420 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:58:55,427 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:58:55,474 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:58:56,426 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:58:57,424 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:58:58,424 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:58:59,425 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:59:00,424 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:59:00,661 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 16:59:00,662 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 16:59:00,662 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 16:59:00,662 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:59:00,664 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 16:59:00,810 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 16:59:01,425 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:59:01,811 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 16:59:02,424 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:59:03,425 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:59:04,425 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:59:05,425 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:59:06,425 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:59:06,470 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:59:07,425 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:59:08,285 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 16:59:08,286 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 16:59:08,425 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:59:09,425 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:59:10,425 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:59:11,425 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:59:12,426 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:59:12,470 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:59:13,426 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:59:14,426 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:59:15,426 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:59:15,535 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 16:59:16,426 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:59:17,426 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:59:17,537 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:59:18,426 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:59:19,426 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:59:20,427 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:59:21,426 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:59:22,427 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:59:23,286 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 16:59:23,286 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 16:59:23,357 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:59:23,427 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:59:24,444 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:59:25,436 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:59:26,436 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:59:27,437 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:59:28,434 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:59:28,476 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:59:29,433 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:59:29,996 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 16:59:29,997 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 16:59:29,997 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 16:59:29,998 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 16:59:30,434 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:59:30,839 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 16:59:31,433 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:59:31,840 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 16:59:32,433 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:59:33,434 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:59:34,434 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:59:34,470 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:59:35,434 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:59:36,434 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:59:37,434 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:59:38,286 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 16:59:38,286 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 16:59:38,434 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:59:39,434 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:59:40,434 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:59:40,456 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:59:41,434 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:59:42,435 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:59:43,435 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:59:44,435 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:59:45,435 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:59:45,471 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:59:45,540 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 16:59:46,435 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:59:47,435 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:59:48,435 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:59:49,435 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:59:50,435 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:59:51,435 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:59:51,471 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:59:52,436 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:59:53,291 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 16:59:53,294 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 16:59:53,442 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:59:54,450 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:59:55,452 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:59:56,450 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:59:56,474 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 16:59:57,449 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:59:58,409 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 16:59:58,411 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 16:59:58,411 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 16:59:58,412 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 16:59:58,844 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:59:58,878 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 16:59:59,555 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 16:59:59,879 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 17:00:00,555 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:00:01,555 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:00:02,473 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:00:02,556 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:00:03,556 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:00:04,556 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:00:05,556 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:00:06,556 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:00:07,473 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:00:07,556 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:00:08,291 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 17:00:08,291 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 17:00:08,556 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:00:09,556 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:00:10,686 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:00:11,559 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:00:12,560 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:00:12,706 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:00:13,559 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:00:14,560 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:00:15,544 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 17:00:15,559 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:00:16,560 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:00:17,560 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:00:18,560 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:00:18,686 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:00:19,560 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:00:20,560 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:00:21,571 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:00:22,568 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:00:23,295 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 17:00:23,295 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 17:00:23,564 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:00:24,403 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:00:24,566 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:00:25,566 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:00:26,566 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:00:27,001 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 17:00:27,002 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 17:00:27,002 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 17:00:27,003 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 17:00:27,566 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:00:27,909 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 17:00:27,910 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 17:00:28,566 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:00:29,566 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:00:30,043 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:00:30,566 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:00:31,566 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:00:32,567 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:00:33,567 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:00:34,567 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:00:35,567 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:00:35,689 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:00:36,567 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:00:37,567 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:00:38,295 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 17:00:38,295 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 17:00:38,567 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:00:39,664 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:00:40,567 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:00:40,689 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:00:41,567 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:00:42,568 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:00:43,568 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:00:44,568 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:00:45,548 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 17:00:45,568 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:00:45,689 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:00:46,568 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:00:47,568 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:00:48,568 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:00:49,568 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:00:50,568 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:00:50,689 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:00:51,568 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:00:52,568 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:00:53,295 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 17:00:53,296 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 17:00:53,568 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:00:54,569 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:00:55,577 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:00:55,697 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:00:56,576 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:00:57,571 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:00:58,572 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:00:59,574 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:01:00,573 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:01:00,917 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 17:01:00,919 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 17:01:00,920 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 17:01:00,920 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:01:00,921 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 17:01:00,937 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 17:01:01,573 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:01:01,938 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 17:01:02,573 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:01:03,573 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:01:04,573 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:01:05,573 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:01:06,573 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:01:06,696 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:01:07,573 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:01:08,295 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 17:01:08,296 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 17:01:08,574 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:01:09,573 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:01:10,574 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:01:11,574 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:01:11,696 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:01:12,574 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:01:13,574 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:01:14,574 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:01:15,551 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 17:01:15,803 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:01:16,798 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:01:16,803 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:01:17,798 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:01:18,798 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:01:19,798 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:01:20,798 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:01:21,798 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:01:22,705 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:01:22,798 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:01:23,295 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 17:01:23,296 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 17:01:23,799 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:01:24,799 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:01:25,799 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:01:26,799 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:01:27,800 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:01:28,709 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:01:28,809 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:01:29,802 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:01:30,804 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:01:31,804 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:01:32,807 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:01:33,710 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:01:33,805 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:01:34,415 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 17:01:34,416 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 17:01:34,417 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 17:01:34,418 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 17:01:34,815 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:01:34,974 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 17:01:35,818 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:01:35,975 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 17:01:36,814 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:01:37,814 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:01:38,295 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 17:01:38,296 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 17:01:38,815 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:01:39,433 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:01:39,814 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:01:40,815 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:01:41,815 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:01:42,815 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:01:43,815 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:01:44,708 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:01:44,815 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:01:45,555 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 17:01:45,815 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:01:46,816 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:01:47,815 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:01:48,815 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:01:49,708 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:01:49,816 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:01:50,816 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:01:51,816 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:01:52,816 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:01:53,295 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 17:01:53,296 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 17:01:53,816 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:01:54,816 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:01:55,373 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:01:55,816 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:01:56,816 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:01:57,816 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:01:58,820 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:01:59,825 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:02:00,711 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:02:00,820 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:02:01,824 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:02:02,829 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:02:03,784 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 17:02:03,786 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 17:02:03,787 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 17:02:03,788 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 17:02:03,997 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 17:02:04,186 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:02:05,185 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:02:05,714 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:02:05,999 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 17:02:06,185 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:02:07,185 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:02:08,185 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:02:08,296 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 17:02:08,296 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 17:02:09,185 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:02:10,185 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:02:11,185 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:02:11,714 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:02:12,186 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:02:13,186 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:02:14,186 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:02:15,186 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:02:15,559 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 17:02:16,186 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:02:16,714 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:02:17,186 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:02:18,186 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:02:19,186 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:02:20,187 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:02:21,187 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:02:21,714 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:02:22,187 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:02:23,187 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:02:23,296 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 17:02:23,296 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 17:02:24,187 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:02:25,187 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:02:26,187 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:02:27,187 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:02:27,715 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:02:28,198 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:02:29,193 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:02:30,190 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:02:31,192 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:02:32,192 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:02:32,724 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:02:33,191 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:02:33,403 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 17:02:33,405 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 17:02:33,405 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 17:02:33,406 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 17:02:34,028 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 17:02:34,029 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 17:02:34,191 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:02:35,190 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:02:36,192 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:02:37,192 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:02:38,192 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:02:38,296 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 17:02:38,296 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 17:02:38,389 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:02:39,192 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:02:40,192 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:02:41,192 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:02:42,192 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:02:43,192 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:02:43,719 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:02:44,192 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:02:45,193 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:02:45,563 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 17:02:46,193 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:02:47,193 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:02:48,193 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:02:48,719 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:02:49,193 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:02:50,193 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:02:51,193 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:02:52,193 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:02:53,193 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:02:53,296 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 17:02:53,296 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 17:02:54,193 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:02:54,403 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:02:55,194 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:02:56,194 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:02:57,203 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:02:58,202 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:02:59,199 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:02:59,721 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:03:00,199 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:03:01,199 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:03:02,200 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:03:02,335 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 17:03:02,337 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 17:03:02,337 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 17:03:02,338 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 17:03:03,058 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 17:03:03,200 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:03:04,059 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 17:03:04,199 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:03:05,200 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:03:05,379 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:03:06,200 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:03:07,200 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:03:08,200 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:03:08,296 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 17:03:08,297 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 17:03:09,200 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:03:10,200 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:03:10,451 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:03:11,200 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:03:12,200 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:03:13,200 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:03:14,365 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:03:15,364 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:03:15,567 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 17:03:15,573 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:03:16,364 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:03:17,364 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:03:18,364 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:03:19,364 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:03:20,365 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:03:20,722 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:03:21,365 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:03:22,365 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:03:23,296 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 17:03:23,297 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 17:03:23,365 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:03:24,374 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:03:25,373 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:03:26,375 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:03:26,724 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:03:27,376 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:03:28,377 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:03:29,377 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:03:29,658 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 17:03:29,659 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 17:03:29,660 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 17:03:29,662 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 17:03:30,089 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 17:03:30,090 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 17:03:30,377 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:03:31,377 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:03:32,377 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:03:32,723 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:03:33,377 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:03:34,377 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:03:35,377 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:03:36,378 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:03:37,378 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:03:38,296 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 17:03:38,297 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 17:03:38,378 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:03:38,464 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:03:39,378 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:03:40,378 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:03:41,378 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:03:42,378 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:03:43,378 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:03:43,723 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:03:44,491 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:03:45,407 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:03:45,571 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 17:03:46,407 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:03:47,407 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:03:48,407 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:03:49,407 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:03:49,723 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:03:50,407 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:03:51,407 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:03:52,407 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:03:53,296 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 17:03:53,297 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 17:03:53,407 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:03:54,407 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:03:55,408 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:03:55,468 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:03:56,408 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:03:57,408 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:03:58,408 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:03:59,419 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:04:00,413 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:04:00,727 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:04:01,413 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:04:02,418 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:04:03,413 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:04:04,413 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:04:04,966 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 17:04:04,967 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 17:04:04,967 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 17:04:04,969 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 17:04:05,117 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 17:04:05,413 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:04:06,118 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 17:04:06,413 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:04:06,726 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:04:07,413 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:04:08,297 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 17:04:08,297 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 17:04:08,413 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:04:09,414 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:04:10,414 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:04:11,414 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:04:12,414 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:04:12,726 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:04:13,414 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:04:14,414 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:04:15,414 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:04:15,575 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 17:04:16,414 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:04:17,414 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:04:18,414 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:04:18,726 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:04:19,415 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:04:20,415 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:04:21,416 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:04:22,415 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:04:23,297 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 17:04:23,297 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 17:04:23,415 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:04:24,415 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:04:24,421 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:04:25,415 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:04:26,415 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:04:27,415 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:04:28,415 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:04:29,415 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:04:29,726 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:04:30,415 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:04:31,416 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:04:32,425 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:04:33,419 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:04:34,419 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:04:34,731 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:04:35,419 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:04:36,419 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:04:37,420 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:04:37,521 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 17:04:37,522 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 17:04:37,523 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 17:04:37,524 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 17:04:38,146 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 17:04:38,146 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 17:04:38,297 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 17:04:38,297 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 17:04:38,420 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:04:39,420 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:04:40,415 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:04:40,420 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:04:41,420 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:04:42,420 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:04:43,420 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:04:44,420 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:04:45,420 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:04:45,580 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 17:04:45,586 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:04:46,420 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:04:47,420 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:04:48,421 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:04:49,420 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:04:50,421 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:04:50,729 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:04:51,421 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:04:52,421 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:04:53,297 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 17:04:53,297 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 17:04:53,421 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:04:54,421 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:04:55,421 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:04:55,730 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:04:56,421 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:04:57,421 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:04:58,421 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:04:59,431 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:05:00,434 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:05:00,731 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:05:01,432 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:05:02,434 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:05:03,434 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:05:04,342 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 17:05:04,343 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 17:05:04,344 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 17:05:04,346 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 17:05:04,740 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:05:05,171 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 17:05:05,489 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:05:05,732 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:05:06,175 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 17:05:06,493 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:05:07,492 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:05:08,297 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 17:05:08,298 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 17:05:08,492 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:05:09,493 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:05:10,493 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:05:11,493 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:05:11,732 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:05:12,493 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:05:13,493 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:05:14,493 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:05:15,493 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:05:15,583 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 17:05:16,493 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:05:17,493 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:05:17,732 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:05:18,493 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:05:19,494 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:05:20,494 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:05:21,494 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:05:22,494 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:05:22,732 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:05:23,297 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 17:05:23,298 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 17:05:23,494 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:05:24,494 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:05:25,494 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:05:26,494 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:05:27,494 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:05:28,494 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:05:28,732 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:05:29,495 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:05:30,495 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:05:31,497 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:05:32,498 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:05:33,497 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:05:33,748 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:05:34,497 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:05:35,182 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 17:05:35,183 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 17:05:35,184 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 17:05:35,185 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 17:05:35,206 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 17:05:35,587 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:05:36,583 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:05:37,208 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 17:05:37,583 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:05:38,297 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 17:05:38,298 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 17:05:38,583 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:05:39,459 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:05:39,583 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:05:40,583 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:05:41,583 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:05:42,584 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:05:43,584 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:05:44,584 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:05:44,738 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:05:45,586 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:05:45,587 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 17:05:46,584 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:05:47,584 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:05:48,584 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:05:49,584 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:05:49,738 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:05:50,584 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:05:51,584 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:05:52,584 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:05:53,297 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 17:05:53,298 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 17:05:53,584 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:05:54,585 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:05:55,388 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:05:55,585 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:05:56,585 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:05:57,585 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:05:58,595 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:05:59,590 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:06:00,590 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:06:00,748 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:06:01,591 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:06:02,591 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:06:03,591 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:06:03,983 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 17:06:03,984 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 17:06:03,985 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 17:06:03,986 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 17:06:04,233 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 17:06:04,592 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:06:05,234 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 17:06:05,591 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:06:06,591 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:06:06,743 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:06:07,591 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:06:08,297 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 17:06:08,298 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 17:06:08,591 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:06:09,591 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:06:10,592 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:06:11,592 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:06:12,592 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:06:12,743 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:06:13,592 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:06:14,592 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:06:15,589 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 17:06:15,596 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:06:16,592 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:06:17,592 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:06:18,592 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:06:18,743 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:06:19,593 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:06:20,593 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:06:21,593 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:06:22,593 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:06:23,298 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 17:06:23,298 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 17:06:23,593 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:06:24,431 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:06:24,593 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:06:25,593 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:06:26,593 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:06:27,593 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:06:28,601 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:06:29,595 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:06:29,749 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:06:30,595 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:06:31,595 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:06:32,597 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:06:33,595 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:06:34,219 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 17:06:34,220 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 17:06:34,220 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 17:06:34,221 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 17:06:34,262 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 17:06:34,620 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:06:35,261 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:06:35,618 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:06:36,618 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:06:37,264 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 17:06:37,618 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:06:38,298 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 17:06:38,298 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 17:06:38,618 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:06:39,618 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:06:40,416 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:06:40,619 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:06:41,619 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:06:42,619 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:06:43,619 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:06:44,619 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:06:45,594 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 17:06:45,600 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:06:45,619 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:06:46,619 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:06:47,619 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:06:48,619 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:06:49,620 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:06:50,620 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:06:50,745 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:06:51,620 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:06:52,620 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:06:53,298 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 17:06:53,298 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 17:06:53,620 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:06:54,620 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:06:55,620 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:06:56,620 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:06:56,745 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:06:57,620 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:06:58,620 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:06:59,620 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:07:00,620 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:07:01,635 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:07:01,747 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:07:02,624 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:07:03,624 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:07:04,625 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:07:05,627 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:07:06,625 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:07:06,757 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:07:06,973 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 17:07:06,974 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 17:07:06,975 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 17:07:06,976 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 17:07:07,291 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 17:07:07,625 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:07:08,298 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 17:07:08,299 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 17:07:08,625 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:07:09,293 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 17:07:09,625 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:07:10,625 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:07:11,625 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:07:12,626 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:07:12,750 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:07:13,626 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:07:14,626 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:07:15,597 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 17:07:15,626 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:07:16,626 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:07:17,626 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:07:18,626 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:07:18,750 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:07:19,626 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:07:20,626 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:07:21,627 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:07:22,627 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:07:23,299 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 17:07:23,299 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 17:07:23,627 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:07:24,395 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:07:24,627 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:07:25,627 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:07:26,627 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:07:27,627 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:07:28,627 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:07:29,627 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:07:29,750 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:07:30,627 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:07:31,627 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:07:32,628 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:07:33,639 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:07:34,630 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:07:34,758 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:07:35,630 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:07:36,630 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:07:37,633 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:07:38,298 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 17:07:38,298 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 17:07:38,633 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:07:39,121 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 17:07:39,122 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 17:07:39,123 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 17:07:39,124 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 17:07:39,326 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 17:07:39,634 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:07:40,163 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:07:40,634 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:07:41,328 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 17:07:41,634 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:07:42,634 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:07:43,634 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:07:44,634 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:07:45,602 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 17:07:45,608 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:07:45,634 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:07:46,634 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:07:47,634 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:07:48,634 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:07:49,634 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:07:50,634 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:07:50,754 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:07:51,634 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:07:52,635 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:07:53,298 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 17:07:53,299 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 17:07:53,635 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:07:54,635 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:07:55,635 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:07:56,635 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:07:56,754 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:07:57,635 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:07:58,635 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:07:59,635 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:08:00,635 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:08:01,635 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:08:02,635 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:08:02,754 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:08:03,636 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:08:04,640 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:08:05,639 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:08:06,639 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:08:07,639 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:08:07,765 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:08:08,299 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 17:08:08,303 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 17:08:08,639 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:08:09,471 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 17:08:09,472 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 17:08:09,472 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 17:08:09,474 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 17:08:09,870 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:08:10,353 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 17:08:10,869 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:08:11,354 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 17:08:11,869 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:08:12,869 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:08:13,758 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:08:13,869 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:08:14,869 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:08:15,605 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 17:08:15,870 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:08:16,870 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:08:17,870 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:08:18,870 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:08:19,758 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:08:19,870 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:08:20,870 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:08:21,870 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:08:22,873 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:08:23,299 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 17:08:23,300 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 17:08:23,872 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:08:24,872 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:08:25,466 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:08:25,872 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:08:26,872 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:08:27,872 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:08:28,872 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:08:29,872 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:08:30,758 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:08:30,872 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:08:31,872 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:08:32,873 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:08:33,873 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:08:34,873 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:08:35,873 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:08:36,762 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:08:36,880 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:08:37,879 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:08:38,302 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 17:08:38,303 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 17:08:38,882 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:08:39,879 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:08:40,879 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:08:41,761 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:08:41,879 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:08:42,450 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 17:08:42,452 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 17:08:42,452 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 17:08:42,454 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 17:08:42,902 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:08:43,383 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 17:08:43,902 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:08:44,902 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:08:45,385 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 17:08:45,609 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 17:08:45,902 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:08:46,762 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:08:46,902 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:08:47,902 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:08:48,902 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:08:49,902 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:08:50,902 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:08:51,902 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:08:52,762 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:08:52,902 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:08:53,301 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 17:08:53,301 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 17:08:53,902 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:08:54,902 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:08:55,902 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:08:56,903 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:08:57,762 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:08:57,903 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:08:58,903 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:08:59,903 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:09:00,903 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:09:01,903 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:09:02,762 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:09:02,903 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:09:03,903 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:09:04,903 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:09:05,903 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:09:06,904 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:09:07,762 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:09:07,908 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:09:08,306 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 17:09:08,318 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 17:09:08,904 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:09:09,904 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:09:10,906 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:09:11,904 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:09:12,768 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:09:12,904 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:09:13,732 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 17:09:13,733 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 17:09:13,733 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 17:09:13,735 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 17:09:14,136 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:09:14,413 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 17:09:15,133 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:09:15,414 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 17:09:15,613 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 17:09:16,133 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:09:17,133 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:09:18,133 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:09:18,763 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:09:19,133 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:09:20,133 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:09:21,134 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:09:22,134 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:09:23,134 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:09:23,304 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 17:09:23,304 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 17:09:24,134 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:09:24,450 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:09:25,134 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:09:26,134 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:09:27,134 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:09:28,135 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:09:29,135 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:09:29,763 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:09:30,134 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:09:31,135 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:09:32,135 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:09:33,135 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:09:34,135 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:09:34,763 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:09:35,135 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:09:36,135 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:09:37,135 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:09:38,135 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:09:38,304 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 17:09:38,304 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 17:09:39,136 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:09:40,135 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:09:40,413 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:09:41,138 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:09:42,143 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:09:43,144 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:09:44,141 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:09:45,141 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:09:45,155 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 17:09:45,156 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 17:09:45,156 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 17:09:45,158 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 17:09:45,445 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 17:09:45,617 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 17:09:45,623 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:09:46,141 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:09:47,141 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:09:47,446 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 17:09:48,142 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:09:49,142 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:09:50,142 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:09:50,769 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:09:51,142 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:09:52,142 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:09:53,142 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:09:53,304 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 17:09:53,304 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 17:09:54,142 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:09:55,142 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:09:56,142 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:09:56,768 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:09:57,142 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:09:58,143 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:09:59,143 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:10:00,143 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:10:01,143 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:10:01,769 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:10:02,143 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:10:03,143 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:10:04,143 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:10:05,143 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:10:06,143 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:10:07,144 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:10:07,769 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:10:08,144 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:10:08,304 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 17:10:08,305 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 17:10:10,666 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:10:11,462 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 17:10:11,761 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:10:13,456 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:10:15,652 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 17:10:18,493 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:10:18,654 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:10:19,912 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:10:23,655 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:10:26,668 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 17:10:26,668 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 17:10:27,597 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:10:29,177 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:10:34,177 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:10:35,054 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:10:36,146 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:10:37,195 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:10:39,558 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:10:44,559 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:10:45,046 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:10:45,305 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 17:10:45,306 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 17:10:45,625 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 17:10:49,627 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:10:52,167 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:10:52,962 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:10:54,472 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:10:55,124 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:11:00,124 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:11:05,125 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:11:05,268 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:11:05,270 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 17:11:05,271 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 17:11:05,310 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:11:06,270 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:11:07,271 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:11:07,700 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 17:11:08,270 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:11:09,271 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:11:09,702 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 17:11:10,271 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:11:10,665 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:11:11,286 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:11:12,285 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:11:13,285 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:11:14,285 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:11:15,285 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:11:15,629 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 17:11:15,968 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:11:15,969 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 17:11:15,969 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 17:11:16,286 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:11:17,286 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:11:18,286 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:11:19,286 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:11:20,287 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:11:21,286 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:11:21,968 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:11:22,286 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:11:23,286 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:11:24,287 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:11:25,287 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:11:26,306 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:11:27,305 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:11:27,968 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:11:28,305 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:11:29,305 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:11:30,306 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:11:30,967 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 17:11:30,968 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 17:11:31,306 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:11:32,306 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:11:33,093 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:11:33,307 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:11:34,306 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:11:35,307 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:11:36,306 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:11:37,306 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:11:38,307 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:11:38,968 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:11:39,307 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:11:40,307 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:11:41,307 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:11:42,307 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:11:43,307 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:11:44,307 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:11:44,969 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:11:45,307 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:11:45,638 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 17:11:45,967 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 17:11:45,968 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 17:11:46,307 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:11:47,307 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:11:48,308 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:11:49,308 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:11:50,049 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:11:50,308 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:11:51,308 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:11:52,308 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:11:53,308 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:11:54,308 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:11:55,308 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:11:55,968 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:11:56,308 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:11:57,309 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:11:58,309 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:11:59,309 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:12:00,969 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:12:05,970 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:12:10,970 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:12:15,637 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 17:12:16,639 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:12:19,861 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 17:12:19,862 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 17:12:19,902 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:12:20,856 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:12:21,856 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:12:21,991 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:12:22,856 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:12:23,855 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:12:24,856 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:12:25,856 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:12:26,856 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:12:27,856 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:12:27,856 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:12:28,856 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:12:29,856 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:12:30,856 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:12:31,856 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:12:32,856 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:12:33,856 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:12:33,856 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:12:34,856 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 17:12:34,856 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 17:12:34,896 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:12:35,857 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:12:36,857 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:12:37,857 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:12:38,857 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:12:39,008 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:12:39,857 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:12:40,857 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:12:41,857 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:12:42,857 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:12:43,858 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:12:44,856 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:12:44,857 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:12:45,641 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 17:12:45,858 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:12:46,858 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:12:47,858 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:12:48,858 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:12:49,856 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 17:12:49,856 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 17:12:49,897 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:12:50,016 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:12:50,858 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:12:51,858 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:12:52,858 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:12:53,858 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:12:54,858 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:12:55,856 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:12:55,859 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:12:56,859 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:12:57,859 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:12:58,859 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:12:59,859 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:13:00,856 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:13:00,859 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:13:01,859 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:13:02,859 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:13:03,859 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:13:04,855 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 17:13:04,856 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 17:13:04,895 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:13:05,860 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:13:05,995 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:13:06,860 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:13:07,860 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:13:08,860 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:13:09,860 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:13:10,860 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:13:11,857 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:13:11,860 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:13:12,860 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:13:13,860 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:13:14,860 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:13:15,646 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 17:13:15,861 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:13:16,861 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:13:17,648 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:13:17,861 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:13:18,861 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:13:19,856 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 17:13:19,856 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 17:13:19,897 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:13:20,861 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:13:21,861 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:13:22,861 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:13:22,977 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:13:23,862 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:13:24,861 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:13:25,862 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:13:26,862 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:13:27,862 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:13:28,858 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:13:28,862 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:13:29,862 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:13:30,862 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:13:31,862 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:13:32,862 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:13:33,862 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:13:34,856 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 17:13:34,856 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 17:13:34,897 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:13:35,000 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:13:35,863 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:13:36,863 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:13:37,863 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:13:38,863 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:13:39,863 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:13:40,858 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:13:40,863 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:13:41,863 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:13:42,863 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:13:43,863 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:13:44,863 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:13:45,649 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 17:13:45,864 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:13:46,652 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:13:46,864 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:13:47,864 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:13:48,864 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:13:49,856 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 17:13:49,856 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 17:13:49,897 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:13:50,864 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:13:51,864 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:13:52,010 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:13:52,864 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:13:53,865 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:13:54,865 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:13:55,865 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:13:56,865 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:13:57,858 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:13:57,865 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:13:58,865 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:13:59,865 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:14:00,865 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:14:01,865 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:14:02,858 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:14:02,865 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:14:03,866 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:14:04,856 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 17:14:04,856 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 17:14:04,896 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:14:05,866 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:14:06,866 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:14:07,866 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:14:08,002 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:14:08,866 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:14:09,866 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:14:10,866 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:14:11,867 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:14:12,867 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:14:13,859 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:14:13,867 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:14:14,867 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:14:15,653 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 17:14:15,867 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:14:16,867 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:14:17,867 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:14:19,656 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:14:24,656 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:14:29,658 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:14:34,668 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:14:39,669 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:14:44,670 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:14:45,657 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 17:14:50,660 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:14:55,661 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:14:57,107 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 17:14:57,108 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 17:14:57,147 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:14:58,107 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:14:59,107 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:15:00,107 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:15:01,110 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:15:01,201 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:15:01,761 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 17:15:02,109 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:15:03,110 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:15:03,762 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 17:15:04,110 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:15:05,110 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:15:06,110 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:15:06,449 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:15:07,110 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:15:08,110 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:15:09,110 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:15:10,110 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:15:11,110 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:15:12,102 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 17:15:12,103 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 17:15:12,143 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:15:12,248 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:15:13,111 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:15:14,111 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:15:15,111 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:15:15,661 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 17:15:16,111 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:15:17,111 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:15:18,108 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:15:18,111 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:15:19,111 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:15:20,111 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:15:21,111 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:15:22,111 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:15:23,108 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:15:23,112 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:15:24,117 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:15:25,118 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:15:26,120 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:15:27,107 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 17:15:27,108 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 17:15:27,148 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:15:28,118 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:15:28,210 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:15:29,118 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:15:29,462 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 17:15:29,463 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 17:15:29,463 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 17:15:29,465 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 17:15:29,791 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 17:15:30,118 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:15:31,124 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:15:31,793 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 17:15:32,118 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:15:33,119 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:15:34,111 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:15:34,119 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:15:35,119 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:15:36,119 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:15:37,119 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:15:38,119 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:15:39,119 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:15:40,111 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:15:40,119 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:15:41,120 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:15:42,106 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 17:15:42,107 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 17:15:42,147 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:15:43,120 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:15:44,120 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:15:45,120 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:15:45,220 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:15:45,666 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 17:15:46,120 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:15:47,120 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:15:48,120 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:15:49,120 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:15:50,120 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:15:51,111 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:15:51,120 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:15:52,120 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:15:53,121 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:15:54,121 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:15:55,121 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:15:56,120 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:15:56,127 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:15:57,123 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 17:15:57,125 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 17:15:57,167 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:15:58,127 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:15:59,131 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:16:00,131 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:16:01,123 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 17:16:01,124 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 17:16:01,124 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 17:16:01,124 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:16:01,125 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 17:16:01,522 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:16:01,817 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 17:16:02,272 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:16:03,274 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:16:03,819 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 17:16:04,273 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:16:05,273 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:16:06,274 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:16:07,113 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:16:07,274 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:16:08,274 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:16:09,274 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:16:10,274 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:16:11,274 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:16:12,110 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 17:16:12,111 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 17:16:12,202 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:16:12,274 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:16:13,274 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:16:14,276 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:16:15,276 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:16:15,669 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 17:16:16,276 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:16:17,276 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:16:18,113 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:16:18,276 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:16:19,276 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:16:20,276 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:16:21,276 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:16:22,276 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:16:23,277 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:16:24,113 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:16:24,276 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:16:25,277 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:16:26,285 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:16:27,110 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 17:16:27,110 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 17:16:27,284 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:16:28,283 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:16:29,185 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:16:29,280 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:16:30,280 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:16:31,280 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:16:32,013 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 17:16:32,014 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 17:16:32,015 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 17:16:32,016 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 17:16:32,411 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:16:32,843 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 17:16:33,408 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:16:33,844 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 17:16:34,408 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:16:34,410 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:16:35,408 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:16:36,408 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:16:37,408 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:16:38,408 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:16:39,408 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:16:40,165 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:16:40,409 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:16:41,409 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:16:42,110 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 17:16:42,111 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 17:16:42,409 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:16:43,409 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:16:44,409 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:16:45,201 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:16:45,503 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:16:45,673 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 17:16:46,502 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:16:47,502 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:16:48,502 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:16:49,502 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:16:50,503 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:16:51,165 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:16:51,503 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:16:52,503 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:16:53,503 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:16:54,503 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:16:55,503 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:16:56,503 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:16:57,111 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 17:16:57,111 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 17:16:57,275 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:16:57,514 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:16:58,509 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:16:59,513 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:17:00,512 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:17:01,513 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:17:02,513 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:17:02,930 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 17:17:02,930 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 17:17:02,931 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 17:17:02,931 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:17:02,932 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 17:17:03,513 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:17:03,868 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 17:17:04,513 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:17:05,513 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:17:05,870 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 17:17:06,513 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:17:07,513 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:17:08,165 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:17:08,514 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:17:09,513 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:17:10,514 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:17:11,514 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:17:12,110 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 17:17:12,111 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 17:17:12,514 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:17:13,214 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:17:13,514 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:17:14,514 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:17:15,515 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:17:15,676 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 17:17:16,514 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:17:17,514 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:17:18,515 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:17:19,166 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:17:19,514 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:17:20,515 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:17:21,515 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:17:22,515 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:17:23,515 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:17:24,515 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:17:25,166 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:17:25,515 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:17:26,515 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:17:27,114 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 17:17:27,114 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 17:17:27,521 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:17:28,519 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:17:29,526 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:17:30,184 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:17:30,526 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:17:31,523 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:17:32,523 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:17:32,870 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 17:17:32,872 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 17:17:32,872 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 17:17:32,873 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 17:17:32,893 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 17:17:33,523 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:17:33,895 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 17:17:34,524 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:17:35,523 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:17:35,913 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:17:36,523 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:17:37,524 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:17:38,524 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:17:39,525 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:17:40,524 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:17:41,169 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:17:41,524 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:17:42,112 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 17:17:42,113 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 17:17:42,524 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:17:43,524 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:17:44,524 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:17:45,524 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:17:45,680 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 17:17:46,525 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:17:46,682 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:17:47,525 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:17:48,525 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:17:49,525 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:17:50,525 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:17:51,525 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:17:52,169 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:17:52,525 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:17:53,525 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:17:54,525 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:17:55,525 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:17:56,530 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:17:57,116 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 17:17:57,119 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 17:17:57,205 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:17:57,529 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:17:58,529 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:17:59,531 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:18:00,531 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:18:01,531 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:18:01,940 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 17:18:01,942 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 17:18:01,942 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 17:18:01,944 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 17:18:02,344 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:18:02,538 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:18:02,919 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 17:18:03,537 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:18:03,920 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 17:18:04,538 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:18:05,538 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:18:06,538 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:18:07,538 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:18:08,343 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:18:08,538 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:18:09,538 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:18:10,538 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:18:11,538 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:18:12,116 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 17:18:12,116 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 17:18:12,538 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:18:13,538 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:18:14,343 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:18:14,546 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:18:15,546 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:18:15,684 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 17:18:16,546 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:18:17,546 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:18:18,546 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:18:19,343 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:18:19,546 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:18:20,546 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:18:21,546 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:18:22,546 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:18:23,547 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:18:24,547 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:18:25,349 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:18:25,553 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:18:26,551 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:18:27,127 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 17:18:27,128 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 17:18:27,558 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:18:28,551 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:18:29,552 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:18:30,347 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:18:30,551 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:18:30,956 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 17:18:30,958 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 17:18:30,958 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 17:18:30,960 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 17:18:31,552 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:18:31,943 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 17:18:31,944 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 17:18:32,552 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:18:33,552 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:18:34,552 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:18:35,552 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:18:36,346 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:18:36,552 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:18:37,552 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:18:38,552 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:18:39,552 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:18:40,552 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:18:41,552 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:18:42,125 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 17:18:42,125 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 17:18:42,271 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:18:42,553 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:18:43,553 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:18:44,553 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:18:45,553 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:18:45,689 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 17:18:46,553 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:18:47,493 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:18:47,553 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:18:48,553 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:18:49,553 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:18:50,553 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:18:51,553 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:18:52,493 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:18:52,554 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:18:53,554 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:18:54,566 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:18:55,562 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:18:56,562 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:18:57,129 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 17:18:57,129 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 17:18:57,503 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:18:57,568 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:18:58,562 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:18:59,563 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:18:59,818 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 17:18:59,820 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 17:18:59,820 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 17:18:59,821 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 17:18:59,968 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 17:19:00,563 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:19:01,562 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:19:01,969 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 17:19:02,564 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:19:03,497 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:19:03,563 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:19:04,563 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:19:05,563 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:19:06,563 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:19:07,563 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:19:08,563 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:19:09,497 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:19:09,563 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:19:10,563 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:19:11,563 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:19:12,129 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 17:19:12,129 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 17:19:12,563 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:19:13,564 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:19:14,707 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:19:15,497 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:19:15,564 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:19:15,692 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 17:19:16,564 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:19:17,564 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:19:18,564 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:19:19,564 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:19:20,497 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:19:20,564 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:19:21,564 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:19:22,564 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:19:23,565 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:19:24,565 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:19:25,497 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:19:25,565 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:19:26,573 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:19:27,135 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 17:19:27,135 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 17:19:27,567 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:19:28,567 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:19:29,567 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:19:30,509 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:19:30,567 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:19:31,568 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:19:31,762 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 17:19:31,764 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 17:19:31,764 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 17:19:31,766 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 17:19:32,001 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 17:19:32,567 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:19:33,568 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:19:34,002 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 17:19:34,568 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:19:35,568 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:19:36,501 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:19:36,568 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:19:37,568 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:19:38,568 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:19:39,568 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:19:40,568 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:19:41,568 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:19:42,135 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 17:19:42,135 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 17:19:42,299 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:19:42,568 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:19:43,568 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:19:44,569 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:19:45,569 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:19:45,696 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 17:19:46,569 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:19:47,501 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:19:47,569 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:19:48,569 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:19:49,569 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:19:50,569 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:19:51,569 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:19:52,501 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:19:52,569 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:19:53,569 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:19:54,570 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:19:55,570 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:19:56,570 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:19:57,135 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 17:19:57,136 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 17:19:57,516 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:19:57,577 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:19:58,577 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:19:59,578 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:20:00,577 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:20:01,577 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:20:02,577 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:20:03,199 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 17:20:03,200 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 17:20:03,201 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 17:20:03,201 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:20:03,202 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 17:20:03,597 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:20:04,029 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 17:20:04,596 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:20:05,596 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:20:06,030 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 17:20:06,596 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:20:07,596 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:20:08,503 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:20:08,596 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:20:09,596 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:20:10,596 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:20:11,596 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:20:12,135 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 17:20:12,136 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 17:20:12,596 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:20:13,596 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:20:14,503 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:20:14,596 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:20:15,596 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:20:15,700 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 17:20:16,597 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:20:17,597 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:20:18,597 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:20:19,503 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:20:19,597 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:20:20,597 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:20:21,597 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:20:22,597 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:20:23,597 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:20:24,598 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:20:25,503 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:20:25,597 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:20:26,597 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:20:27,135 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 17:20:27,136 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 17:20:27,598 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:20:28,602 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:20:29,602 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:20:30,514 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:20:30,604 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:20:31,605 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:20:32,602 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:20:33,501 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 17:20:33,502 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 17:20:33,502 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 17:20:33,504 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 17:20:33,902 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:20:34,051 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 17:20:34,900 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:20:35,542 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:20:35,900 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:20:36,052 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 17:20:36,900 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:20:37,900 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:20:38,900 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:20:39,900 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:20:40,900 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:20:41,507 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:20:41,900 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:20:42,135 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 17:20:42,136 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 17:20:42,901 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:20:43,901 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:20:44,901 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:20:45,704 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 17:20:45,901 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:20:46,706 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:20:46,901 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:20:47,901 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:20:48,901 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:20:49,901 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:20:50,901 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:20:51,901 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:20:52,507 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:20:52,901 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:20:53,901 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:20:54,902 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:20:55,902 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:20:56,902 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:20:57,135 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 17:20:57,136 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 17:20:57,507 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:20:57,910 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:20:58,912 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:20:59,915 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:21:00,912 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:21:01,913 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:21:02,515 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:21:02,913 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:21:03,570 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 17:21:03,571 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 17:21:03,571 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 17:21:03,573 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 17:21:03,973 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:21:04,086 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 17:21:04,971 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:21:05,971 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:21:06,087 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 17:21:06,971 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:21:07,972 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:21:08,510 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:21:08,972 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:21:09,972 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:21:10,972 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:21:11,972 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:21:12,136 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 17:21:12,136 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 17:21:12,972 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:21:13,972 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:21:14,510 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:21:14,972 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:21:15,709 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 17:21:15,972 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:21:16,972 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:21:17,973 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:21:18,973 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:21:19,510 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:21:19,973 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:21:20,973 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:21:21,973 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:21:22,973 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:21:23,973 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:21:24,510 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:21:24,973 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:21:25,973 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:21:26,974 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:21:27,136 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 17:21:27,136 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 17:21:27,974 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:21:28,974 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:21:29,974 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:21:30,510 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:21:30,974 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:21:31,974 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:21:32,985 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:21:33,978 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:21:34,978 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:21:35,512 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:21:35,979 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:21:36,981 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:21:37,980 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:21:38,394 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 17:21:38,396 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 17:21:38,396 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 17:21:38,398 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 17:21:38,980 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:21:39,117 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 17:21:39,980 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:21:40,118 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 17:21:40,980 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:21:41,435 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:21:41,980 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:21:42,136 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 17:21:42,136 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 17:21:42,980 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:21:43,980 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:21:44,980 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:21:45,712 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 17:21:45,980 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:21:46,714 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:21:46,980 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:21:47,981 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:21:48,981 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:21:50,122 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:21:51,030 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:21:52,030 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:21:52,511 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:21:53,030 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:21:54,030 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:21:55,030 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:21:56,030 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:21:57,031 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:21:57,136 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 17:21:57,137 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 17:21:58,031 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:21:58,511 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:21:59,031 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:22:00,031 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:22:01,037 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:22:02,040 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:22:03,040 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:22:03,519 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:22:04,037 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:22:05,040 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:22:05,927 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 17:22:05,928 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 17:22:05,929 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 17:22:05,930 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 17:22:06,145 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 17:22:06,330 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:22:07,078 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:22:08,078 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:22:08,146 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 17:22:09,078 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:22:09,515 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:22:10,078 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:22:11,078 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:22:12,079 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:22:12,136 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 17:22:12,137 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 17:22:13,078 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:22:14,079 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:22:15,079 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:22:15,515 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:22:15,716 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 17:22:16,079 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:22:17,079 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:22:18,079 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:22:19,079 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:22:20,079 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:22:20,515 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:22:21,079 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:22:22,079 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:22:23,079 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:22:24,080 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:22:25,080 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:22:26,080 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:22:26,515 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:22:27,080 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:22:27,136 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 17:22:27,137 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 17:22:28,080 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:22:29,080 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:22:30,086 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:22:31,088 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:22:31,522 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:22:32,085 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:22:33,085 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:22:34,086 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:22:35,087 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:22:35,257 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 17:22:35,259 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 17:22:35,259 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 17:22:35,260 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 17:22:36,087 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:22:36,175 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 17:22:36,176 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 17:22:37,087 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:22:37,300 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:22:38,087 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:22:39,087 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:22:40,087 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:22:41,087 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:22:42,087 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:22:42,136 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 17:22:42,137 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 17:22:42,516 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:22:43,087 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:22:44,087 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:22:45,088 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:22:45,720 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 17:22:46,088 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:22:47,088 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:22:48,088 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:22:48,516 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:22:49,088 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:22:50,088 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:22:51,088 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:22:52,088 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:22:53,088 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:22:53,516 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:22:54,089 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:22:55,089 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:22:56,089 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:22:57,089 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:22:57,136 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 17:22:57,137 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 17:22:58,089 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:22:59,089 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:22:59,516 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:23:00,089 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:23:01,089 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:23:02,100 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:23:03,095 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:23:04,093 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:23:04,517 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:23:05,093 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:23:06,093 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:23:07,094 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:23:07,442 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 17:23:07,444 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 17:23:07,444 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 17:23:07,446 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 17:23:08,093 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:23:08,200 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 17:23:09,094 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:23:09,841 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:23:10,094 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:23:10,202 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 17:23:11,094 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:23:12,094 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:23:12,136 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 17:23:12,137 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 17:23:13,094 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:23:14,094 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:23:15,094 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:23:15,589 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:23:15,724 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 17:23:16,094 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:23:17,094 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:23:18,094 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:23:19,095 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:23:20,095 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:23:21,095 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:23:21,589 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:23:22,095 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:23:23,095 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:23:24,095 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:23:25,095 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:23:26,095 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:23:27,095 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:23:27,137 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 17:23:27,137 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 17:23:27,258 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:23:28,095 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:23:29,096 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:23:30,096 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:23:31,096 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:23:32,096 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:23:32,589 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:23:33,096 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:23:34,096 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:23:35,096 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:23:36,102 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:23:37,103 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:23:37,595 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:23:38,108 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:23:39,107 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:23:40,107 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:23:41,107 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:23:41,141 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 17:23:41,142 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 17:23:41,142 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 17:23:41,143 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 17:23:41,231 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 17:23:42,232 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 17:23:42,264 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 17:23:42,264 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 17:23:42,270 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:23:42,595 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:23:43,263 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:23:44,264 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:23:45,263 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:23:45,728 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 17:23:46,263 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:23:47,263 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:23:48,263 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:23:48,595 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:23:49,263 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:23:50,264 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:23:51,263 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:23:52,263 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:23:53,263 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:23:53,595 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:23:54,264 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:23:55,264 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:23:56,264 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:23:57,262 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 17:23:57,263 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 17:23:57,302 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:23:58,264 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:23:59,264 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:23:59,595 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:24:00,264 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:24:01,264 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:24:02,264 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:24:03,264 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:24:04,264 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:24:04,595 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:24:05,271 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:24:06,265 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:24:07,268 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:24:08,265 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:24:09,269 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:24:09,618 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:24:10,269 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:24:10,954 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 17:24:10,955 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 17:24:10,955 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 17:24:10,956 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 17:24:11,259 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 17:24:12,260 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 17:24:12,275 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 17:24:12,276 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 17:24:12,282 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:24:13,273 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:24:14,273 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:24:15,274 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:24:15,599 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:24:15,733 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 17:24:16,274 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:24:17,274 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:24:18,274 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:24:19,274 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:24:20,274 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:24:21,274 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:24:21,599 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:24:22,274 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:24:23,274 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:24:24,275 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:24:25,274 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:24:26,275 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:24:26,599 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:24:27,274 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 17:24:27,274 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 17:24:27,314 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:24:28,275 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:24:29,275 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:24:30,275 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:24:31,275 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:24:31,599 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:24:32,275 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:24:33,275 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:24:34,275 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:24:35,275 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:24:36,275 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:24:36,609 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:24:37,279 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:24:38,279 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:24:39,283 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:24:40,283 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:24:41,284 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:24:41,449 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 17:24:41,450 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 17:24:41,451 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 17:24:41,452 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 17:24:42,294 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 17:24:42,294 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 17:24:42,492 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:24:42,603 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 17:24:42,604 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 17:24:42,609 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:24:43,602 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:24:44,602 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:24:45,602 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:24:45,736 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 17:24:46,602 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:24:47,603 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:24:47,604 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:24:48,602 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:24:49,602 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:24:50,603 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:24:51,603 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:24:52,603 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:24:53,603 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:24:53,603 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:24:54,603 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:24:55,603 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:24:56,603 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:24:57,602 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 17:24:57,602 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 17:24:57,643 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:24:58,603 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:24:58,674 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:24:59,604 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:25:00,604 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:25:01,604 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:25:02,604 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:25:03,604 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:25:04,603 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:25:04,604 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:25:05,604 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:25:06,609 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:25:07,614 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:25:08,613 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:25:09,616 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:25:09,620 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:25:10,619 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:25:11,617 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:25:12,178 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 17:25:12,179 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 17:25:12,179 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 17:25:12,181 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 17:25:12,322 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 17:25:12,844 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 17:25:12,845 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 17:25:12,851 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:25:13,841 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:25:14,324 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 17:25:14,841 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:25:14,994 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:25:15,740 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 17:25:15,841 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:25:16,842 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:25:17,842 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:25:18,842 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:25:19,842 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:25:20,842 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:25:20,843 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:25:21,842 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:25:22,842 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:25:23,842 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:25:24,842 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:25:25,890 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:25:26,843 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:25:26,889 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:25:27,841 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 17:25:27,841 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 17:25:27,889 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:25:28,889 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:25:29,889 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:25:30,889 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:25:31,889 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:25:32,006 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:25:32,889 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:25:33,889 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:25:34,889 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:25:35,890 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:25:36,890 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:25:37,843 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:25:37,890 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:25:38,895 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:25:39,899 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:25:40,902 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:25:41,904 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:25:42,844 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 17:25:42,844 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 17:25:42,906 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:25:43,010 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:25:43,912 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:25:44,706 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 17:25:44,708 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 17:25:44,708 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 17:25:44,710 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 17:25:45,114 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:25:45,358 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 17:25:45,744 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 17:25:46,111 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:25:46,359 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 17:25:47,111 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:25:48,111 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:25:48,849 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:25:49,111 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:25:50,111 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:25:51,111 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:25:52,111 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:25:53,112 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:25:54,112 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:25:54,849 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:25:55,112 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:25:56,112 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:25:57,112 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:25:57,844 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 17:25:57,845 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 17:25:58,112 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:25:59,112 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:25:59,934 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:26:00,112 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:26:01,112 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:26:02,112 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:26:03,112 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:26:04,113 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:26:05,113 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:26:05,849 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:26:06,113 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:26:07,113 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:26:08,113 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:26:09,113 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:26:10,124 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:26:10,854 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:26:11,119 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:26:12,119 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:26:12,847 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 17:26:12,848 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 17:26:13,128 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:26:14,121 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:26:15,121 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:26:15,354 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 17:26:15,355 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 17:26:15,356 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 17:26:15,358 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 17:26:15,385 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 17:26:15,747 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 17:26:16,124 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:26:16,386 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 17:26:16,749 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:26:17,121 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:26:18,122 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:26:19,121 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:26:20,122 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:26:21,122 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:26:21,853 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:26:22,122 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:26:23,122 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:26:24,122 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:26:25,122 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:26:26,122 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:26:27,122 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:26:27,846 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 17:26:27,847 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 17:26:27,937 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:26:28,122 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:26:29,122 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:26:30,123 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:26:31,123 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:26:32,123 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:26:33,123 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:26:33,853 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:26:34,123 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:26:35,123 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:26:36,123 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:26:37,123 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:26:38,123 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:26:39,124 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:26:39,861 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:26:40,135 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:26:41,139 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:26:42,142 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:26:42,847 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 17:26:42,850 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 17:26:43,142 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:26:44,142 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:26:44,930 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:26:45,143 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:26:45,292 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 17:26:45,293 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 17:26:45,294 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 17:26:45,296 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 17:26:45,414 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 17:26:45,752 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 17:26:46,142 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:26:46,415 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 17:26:47,142 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:26:48,142 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:26:49,143 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:26:50,143 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:26:50,854 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:26:51,143 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:26:52,143 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:26:53,143 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:26:54,143 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:26:55,143 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:26:56,143 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:26:56,854 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:26:57,143 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:26:57,847 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 17:26:57,848 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 17:26:58,143 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:26:59,143 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:27:00,144 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:27:01,144 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:27:01,980 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:27:02,144 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:27:03,144 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:27:04,144 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:27:05,144 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:27:06,144 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:27:07,149 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:27:07,860 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:27:08,151 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:27:09,151 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:27:10,151 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:27:11,151 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:27:12,151 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:27:12,538 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 17:27:12,539 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 17:27:12,540 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 17:27:12,541 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 17:27:12,945 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 17:27:12,946 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 17:27:13,040 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:27:13,152 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:27:13,441 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 17:27:14,152 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:27:14,441 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 17:27:15,152 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:27:15,755 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 17:27:16,152 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:27:17,152 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:27:18,152 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:27:18,944 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:27:19,152 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:27:20,152 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:27:21,152 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:27:22,152 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:27:23,152 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:27:24,152 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:27:24,944 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:27:25,152 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:27:26,153 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:27:27,171 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:27:27,945 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 17:27:27,945 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 17:27:28,169 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:27:29,169 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:27:30,041 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:27:30,169 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:27:31,169 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:27:32,170 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:27:33,170 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:27:34,170 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:27:35,170 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:27:35,945 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:27:36,170 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:27:37,170 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:27:38,170 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:27:39,176 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:27:40,179 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:27:40,950 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:27:41,176 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:27:42,176 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:27:42,946 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 17:27:42,946 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 17:27:43,176 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:27:44,176 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:27:44,870 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 17:27:44,871 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 17:27:44,871 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 17:27:44,872 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 17:27:45,272 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:27:45,469 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 17:27:45,759 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 17:27:46,272 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:27:46,470 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 17:27:46,761 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:27:47,272 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:27:48,272 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:27:49,272 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:27:50,272 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:27:51,273 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:27:51,948 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:27:52,272 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:27:53,272 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:27:54,273 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:27:55,273 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:27:56,273 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:27:57,273 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:27:57,948 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:27:58,047 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 17:27:58,048 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 17:27:58,273 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:27:59,273 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:28:00,273 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:28:01,273 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:28:02,273 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:28:03,047 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:28:03,273 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:28:04,273 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:28:05,274 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:28:06,274 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:28:07,274 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:28:08,274 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:28:09,047 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:28:09,274 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:28:10,274 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:28:11,274 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:28:12,278 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:28:12,959 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 17:28:12,959 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 17:28:13,281 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:28:14,133 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:28:14,274 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:28:15,276 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:28:15,765 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 17:28:16,276 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:28:16,783 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 17:28:16,784 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 17:28:16,784 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 17:28:16,785 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 17:28:17,275 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:28:17,498 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 17:28:18,275 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:28:18,499 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 17:28:19,275 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:28:20,053 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:28:20,275 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:28:21,275 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:28:22,275 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:28:23,275 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:28:24,275 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:28:25,275 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:28:26,053 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:28:26,276 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:28:27,276 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:28:27,959 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 17:28:27,959 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 17:28:28,276 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:28:29,276 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:28:30,276 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:28:31,087 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:28:31,276 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:28:32,276 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:28:33,276 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:28:34,276 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:28:35,276 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:28:36,277 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:28:37,053 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:28:37,277 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:28:38,277 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:28:39,277 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:28:40,277 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:28:41,277 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:28:42,059 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:28:42,279 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:28:42,964 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 17:28:42,964 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 17:28:43,282 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:28:44,282 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:28:45,282 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:28:45,766 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 17:28:46,285 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:28:47,285 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:28:47,320 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 17:28:47,322 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 17:28:47,322 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 17:28:47,322 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:28:47,323 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 17:28:47,528 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 17:28:48,285 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:28:48,529 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 17:28:49,285 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:28:50,285 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:28:51,285 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:28:52,285 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:28:53,058 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:28:53,285 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:28:54,285 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:28:55,285 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:28:56,286 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:28:57,286 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:28:57,964 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 17:28:57,964 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 17:28:58,129 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:28:58,286 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:28:59,286 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:29:00,286 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:29:01,286 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:29:02,286 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:29:03,286 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:29:04,058 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:29:04,286 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:29:05,286 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:29:06,287 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:29:07,287 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:29:08,287 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:29:09,058 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:29:09,287 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:29:10,287 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:29:11,287 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:29:12,299 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:29:12,972 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 17:29:12,972 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 17:29:13,292 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:29:14,073 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:29:14,292 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:29:15,292 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:29:15,773 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 17:29:16,293 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:29:17,294 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:29:17,452 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 17:29:17,453 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 17:29:17,453 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 17:29:17,455 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 17:29:17,559 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 17:29:18,293 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:29:18,560 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 17:29:19,293 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:29:20,062 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:29:20,293 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:29:21,293 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:29:22,293 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:29:23,294 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:29:24,294 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:29:25,294 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:29:26,062 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:29:26,294 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:29:27,294 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:29:27,972 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 17:29:27,972 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 17:29:28,294 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:29:29,294 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:29:30,294 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:29:31,294 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:29:32,062 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:29:32,294 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:29:33,294 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:29:34,295 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:29:35,295 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:29:36,295 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:29:37,062 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:29:37,295 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:29:38,295 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:29:39,295 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:29:40,295 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:29:41,299 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:29:42,068 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:29:42,298 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:29:42,973 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 17:29:42,975 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 17:29:43,300 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:29:44,301 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:29:45,301 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:29:45,774 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 17:29:46,301 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:29:46,966 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 17:29:46,967 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 17:29:46,967 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 17:29:46,969 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 17:29:47,371 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:29:47,588 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 17:29:48,007 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:29:48,370 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:29:48,589 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 17:29:49,370 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:29:50,370 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:29:51,370 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:29:52,371 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:29:53,067 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:29:53,371 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:29:54,371 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:29:55,371 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:29:56,371 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:29:57,371 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:29:57,973 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 17:29:57,974 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 17:29:58,371 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:29:59,067 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:29:59,371 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:30:00,372 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:30:01,372 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:30:02,372 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:30:03,372 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:30:04,067 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:30:04,372 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:30:05,372 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:30:06,372 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:30:07,372 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:30:08,372 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:30:09,372 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:30:10,067 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:30:10,372 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:30:11,373 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:30:12,373 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:30:12,973 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 17:30:12,980 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 17:30:13,378 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:30:14,381 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:30:15,170 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:30:15,382 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:30:15,779 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 17:30:16,385 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:30:17,383 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:30:17,553 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 17:30:17,554 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 17:30:17,554 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 17:30:17,556 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 17:30:17,613 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 17:30:18,382 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:30:18,614 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 17:30:19,382 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:30:20,382 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:30:21,068 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:30:21,383 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:30:22,383 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:30:23,383 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:30:24,383 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:30:25,383 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:30:26,383 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:30:27,068 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:30:27,383 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:30:27,973 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 17:30:27,974 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 17:30:28,383 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:30:29,383 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:30:30,383 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:30:31,384 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:30:32,132 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:30:32,383 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:30:33,383 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:30:34,383 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:30:35,384 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:30:36,384 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:30:37,384 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:30:38,068 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:30:38,384 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:30:39,384 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:30:40,384 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:30:41,402 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:30:42,408 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:30:42,977 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 17:30:42,977 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 17:30:43,073 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:30:43,408 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:30:44,408 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:30:45,408 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:30:45,780 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 17:30:46,408 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:30:46,609 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 17:30:46,611 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 17:30:46,612 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 17:30:46,617 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 17:30:46,635 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 17:30:47,408 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:30:48,408 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:30:48,637 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 17:30:49,073 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:30:49,408 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:30:50,408 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:30:51,409 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:30:52,409 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:30:53,409 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:30:54,073 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:30:54,409 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:30:55,409 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:30:56,409 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:30:57,409 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:30:57,977 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 17:30:57,977 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 17:30:58,409 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:30:59,145 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:30:59,409 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:31:00,410 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:31:01,410 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:31:02,410 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:31:03,410 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:31:04,410 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:31:05,073 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:31:05,410 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:31:06,410 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:31:07,410 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:31:08,410 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:31:09,410 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:31:10,411 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:31:11,073 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:31:11,411 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:31:12,411 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:31:12,977 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 17:31:12,978 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 17:31:13,415 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:31:14,415 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:31:15,423 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:31:15,781 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 17:31:16,422 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:31:16,782 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:31:17,416 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:31:18,416 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:31:19,114 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 17:31:19,116 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 17:31:19,117 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 17:31:19,122 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 17:31:19,514 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:31:19,664 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 17:31:20,516 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:31:20,665 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 17:31:21,515 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:31:22,156 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:31:22,515 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:31:23,515 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:31:24,516 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:31:25,515 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:31:26,516 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:31:27,516 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:31:27,977 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 17:31:27,978 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 17:31:28,067 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:31:28,516 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:31:29,516 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:31:30,516 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:31:31,516 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:31:32,516 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:31:33,077 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:31:33,516 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:31:34,517 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:31:35,517 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:31:36,517 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:31:37,517 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:31:38,517 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:31:39,077 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:31:39,517 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:31:40,517 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:31:41,524 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:31:42,524 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:31:42,982 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 17:31:42,983 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 17:31:43,528 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:31:44,085 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:31:44,530 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:31:45,528 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:31:45,792 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 17:31:46,528 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:31:47,108 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 17:31:47,109 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 17:31:47,109 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 17:31:47,110 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 17:31:47,528 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:31:47,689 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 17:31:48,528 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:31:48,690 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 17:31:49,085 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:31:49,528 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:31:50,528 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:31:51,529 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:31:52,529 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:31:53,529 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:31:54,529 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:31:55,085 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:31:55,529 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:31:56,529 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:31:57,529 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:31:57,979 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 17:31:57,979 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 17:31:58,529 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:31:59,529 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:32:00,117 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:32:00,529 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:32:01,530 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:32:02,530 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:32:03,530 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:32:04,530 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:32:05,530 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:32:06,085 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:32:06,530 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:32:07,530 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:32:08,530 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:32:09,530 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:32:10,531 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:32:11,531 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:32:12,085 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:32:12,531 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:32:12,989 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 17:32:12,992 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 17:32:13,534 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:32:14,534 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:32:15,534 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:32:15,799 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 17:32:16,534 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:32:17,534 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:32:17,800 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:32:18,344 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 17:32:18,346 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 17:32:18,346 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 17:32:18,348 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 17:32:18,719 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 17:32:18,753 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:32:19,751 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:32:20,751 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:32:21,721 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 17:32:21,751 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:32:22,751 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:32:23,088 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:32:23,751 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:32:24,751 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:32:25,751 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:32:26,751 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:32:27,751 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:32:27,979 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 17:32:27,980 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 17:32:28,112 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:32:28,751 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:32:29,751 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:32:30,752 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:32:31,752 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:32:32,753 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:32:33,752 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:32:34,088 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:32:34,752 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:32:35,752 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:32:36,752 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:32:37,752 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:32:38,753 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:32:39,753 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:32:40,088 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:32:40,753 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:32:41,753 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:32:42,753 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:32:42,979 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 17:32:42,980 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 17:32:43,753 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:32:44,760 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:32:45,152 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:32:45,756 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:32:45,793 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 17:32:46,757 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:32:47,757 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:32:48,757 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:32:49,717 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 17:32:49,718 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 17:32:49,718 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 17:32:49,723 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 17:32:49,749 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 17:32:50,117 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:32:50,758 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:32:50,863 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:32:51,750 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 17:32:51,863 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:32:52,863 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:32:53,864 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:32:54,864 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:32:55,864 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:32:56,090 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:32:56,864 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:32:57,864 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:32:57,979 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 17:32:57,980 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 17:32:58,864 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:32:59,864 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:33:00,864 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:33:01,148 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:33:01,864 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:33:02,865 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:33:03,864 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:33:04,865 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:33:05,865 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:33:06,865 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:33:07,091 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:33:07,865 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:33:08,865 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:33:09,865 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:33:10,865 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:33:11,865 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:33:12,865 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:33:12,979 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 17:33:12,980 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 17:33:13,136 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:33:13,865 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:33:14,877 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:33:15,797 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 17:33:15,875 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:33:16,871 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:33:17,871 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:33:18,871 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:33:19,097 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:33:19,871 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:33:20,213 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 17:33:20,214 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 17:33:20,215 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 17:33:20,219 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 17:33:20,778 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 17:33:20,872 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:33:21,778 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 17:33:21,872 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:33:22,872 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:33:23,872 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:33:24,872 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:33:25,095 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:33:25,872 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:33:26,872 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:33:27,872 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:33:27,980 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 17:33:27,980 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 17:33:28,872 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:33:29,872 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:33:30,100 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:33:30,872 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:33:31,872 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:33:32,872 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:33:33,873 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:33:34,873 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:33:35,873 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:33:36,095 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:33:36,873 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:33:37,873 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:33:38,873 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:33:39,873 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:33:40,873 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:33:41,873 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:33:42,095 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:33:42,873 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:33:42,980 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 17:33:42,980 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 17:33:43,873 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:33:44,874 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:33:45,799 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 17:33:45,874 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:33:46,874 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:33:47,801 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:33:47,886 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:33:48,879 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:33:49,882 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:33:50,882 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:33:51,882 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:33:52,886 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:33:53,097 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:33:53,339 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 17:33:53,341 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 17:33:53,341 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 17:33:53,343 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 17:33:53,806 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 17:33:53,883 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:33:54,886 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:33:55,807 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 17:33:55,883 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:33:56,883 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:33:57,883 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:33:57,980 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 17:33:57,981 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 17:33:58,144 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:33:58,883 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:33:59,883 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:34:00,883 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:34:01,883 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:34:02,883 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:34:03,883 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:34:04,098 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:34:04,883 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:34:05,884 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:34:06,884 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:34:07,884 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:34:08,884 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:34:09,098 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:34:09,884 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:34:10,884 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:34:11,884 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:34:12,884 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:34:12,980 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 17:34:12,981 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 17:34:13,884 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:34:14,884 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:34:15,098 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:34:15,803 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 17:34:15,884 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:34:16,885 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:34:17,885 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:34:18,898 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:34:19,888 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:34:20,099 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:34:20,889 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:34:21,889 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:34:22,893 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:34:23,893 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:34:24,599 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 17:34:24,601 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 17:34:24,601 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 17:34:24,603 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 17:34:24,838 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 17:34:25,004 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:34:25,640 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:34:25,840 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 17:34:26,004 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:34:27,004 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:34:27,980 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 17:34:27,981 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 17:34:28,020 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:34:29,004 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:34:30,004 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:34:31,004 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:34:31,100 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:34:32,004 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:34:33,004 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:34:34,004 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:34:35,004 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:34:36,004 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:34:36,100 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:34:37,063 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:34:38,063 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:34:39,062 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:34:40,063 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:34:41,063 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:34:41,100 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:34:42,063 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:34:42,980 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 17:34:42,981 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 17:34:43,063 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:34:44,063 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:34:45,063 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:34:45,807 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 17:34:46,063 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:34:46,809 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:34:47,063 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:34:48,064 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:34:49,070 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:34:50,066 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:34:51,066 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:34:52,066 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:34:52,106 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:34:53,066 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:34:53,913 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 17:34:53,914 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 17:34:53,915 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 17:34:53,916 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 17:34:54,315 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:34:54,863 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 17:34:55,314 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:34:55,864 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 17:34:56,314 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:34:57,314 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:34:57,956 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:34:57,980 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 17:34:57,981 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 17:34:58,314 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:34:59,314 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:35:00,314 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:35:01,314 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:35:02,315 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:35:03,102 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:35:03,315 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:35:04,315 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:35:05,315 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:35:06,315 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:35:07,315 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:35:08,102 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:35:08,316 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:35:09,315 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:35:10,316 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:35:11,316 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:35:12,316 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:35:12,981 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 17:35:12,981 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 17:35:13,316 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:35:14,102 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:35:14,316 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:35:15,316 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:35:15,811 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 17:35:16,316 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:35:17,316 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:35:18,316 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:35:19,102 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:35:19,316 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:35:20,325 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:35:21,321 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:35:22,325 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:35:23,325 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:35:24,107 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:35:24,325 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:35:25,325 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:35:26,048 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 17:35:26,050 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 17:35:26,051 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 17:35:26,054 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 17:35:26,426 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:35:26,892 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 17:35:27,425 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:35:27,893 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 17:35:27,981 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 17:35:27,981 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 17:35:28,425 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:35:29,425 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:35:30,105 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:35:30,425 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:35:31,425 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:35:32,425 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:35:33,425 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:35:34,425 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:35:35,105 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:35:35,425 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:35:36,425 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:35:37,426 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:35:38,426 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:35:39,426 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:35:40,105 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:35:40,426 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:35:41,426 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:35:42,426 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:35:42,981 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 17:35:42,981 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 17:35:43,426 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:35:44,426 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:35:45,119 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:35:45,426 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:35:45,815 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 17:35:46,426 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:35:47,427 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:35:48,427 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:35:49,427 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:35:50,427 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:35:51,106 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:35:51,427 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:35:52,428 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:35:53,431 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:35:54,438 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:35:55,432 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:35:56,115 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:35:56,435 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:35:57,432 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:35:57,981 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 17:35:57,981 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 17:35:58,225 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 17:35:58,226 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 17:35:58,226 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 17:35:58,229 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 17:35:58,632 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:35:58,919 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 17:35:59,628 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:35:59,920 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 17:36:00,628 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:36:01,268 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:36:01,628 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:36:02,628 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:36:03,628 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:36:04,628 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:36:05,628 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:36:06,628 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:36:07,111 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:36:07,628 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:36:08,628 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:36:09,628 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:36:10,629 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:36:11,629 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:36:12,629 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:36:12,981 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 17:36:12,982 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 17:36:13,128 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:36:13,629 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:36:14,629 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:36:15,629 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:36:15,819 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 17:36:16,629 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:36:17,629 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:36:18,629 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:36:19,111 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:36:19,629 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:36:20,629 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:36:21,630 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:36:22,640 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:36:23,635 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:36:24,118 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:36:24,635 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:36:25,639 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:36:26,641 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:36:27,572 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 17:36:27,573 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 17:36:27,573 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 17:36:27,574 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 17:36:27,943 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 17:36:27,973 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:36:27,981 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 17:36:27,982 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 17:36:28,972 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:36:29,945 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 17:36:29,972 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:36:30,111 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:36:30,972 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:36:31,972 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:36:32,972 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:36:33,973 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:36:34,973 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:36:35,112 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:36:35,973 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:36:36,973 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:36:37,973 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:36:38,973 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:36:39,973 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:36:41,112 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:36:41,337 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:36:42,325 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:36:42,981 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 17:36:42,982 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 17:36:43,325 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:36:44,325 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:36:45,326 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:36:45,823 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 17:36:46,326 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:36:46,825 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:36:47,326 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:36:48,326 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:36:49,326 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:36:50,326 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:36:51,326 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:36:52,112 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:36:52,326 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:36:53,328 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:36:54,334 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:36:55,331 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:36:56,336 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:36:57,119 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:36:57,331 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:36:57,984 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 17:36:57,984 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 17:36:58,331 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:36:59,332 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:36:59,615 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 17:36:59,616 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 17:36:59,616 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 17:36:59,618 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 17:36:59,978 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 17:37:00,332 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:37:01,339 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:37:01,980 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 17:37:02,332 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:37:02,657 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:37:03,332 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:37:04,332 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:37:05,332 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:37:06,332 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:37:07,332 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:37:08,116 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:37:08,332 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:37:09,332 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:37:10,333 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:37:11,333 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:37:12,552 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:37:12,984 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 17:37:12,985 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 17:37:13,464 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:37:14,116 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:37:14,464 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:37:15,464 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:37:15,827 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 17:37:16,464 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:37:17,464 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:37:18,464 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:37:19,116 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:37:19,464 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:37:20,464 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:37:21,465 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:37:22,465 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:37:23,465 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:37:24,465 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:37:25,121 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:37:25,477 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:37:26,471 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:37:27,474 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:37:27,984 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 17:37:27,984 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 17:37:28,474 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:37:29,471 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:37:30,471 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:37:30,757 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 17:37:30,759 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 17:37:30,760 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 17:37:30,760 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:37:30,766 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 17:37:31,008 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 17:37:31,472 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:37:32,009 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 17:37:32,472 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:37:33,472 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:37:34,472 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:37:35,472 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:37:36,117 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:37:36,472 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:37:37,472 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:37:38,472 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:37:39,472 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:37:40,472 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:37:41,472 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:37:42,117 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:37:42,472 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:37:43,101 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 17:37:43,101 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 17:37:43,473 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:37:44,473 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:37:45,473 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:37:45,832 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 17:37:46,473 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:37:47,473 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:37:47,834 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:37:48,473 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:37:49,473 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:37:50,473 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:37:51,473 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:37:52,473 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:37:53,117 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:37:53,474 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:37:54,474 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:37:55,479 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:37:56,478 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:37:57,478 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:37:58,023 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 17:37:58,023 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 17:37:58,127 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:37:58,478 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:37:59,480 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:38:00,479 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:38:00,930 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 17:38:00,931 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 17:38:00,932 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 17:38:00,934 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 17:38:01,035 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 17:38:01,479 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:38:02,036 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 17:38:02,478 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:38:03,479 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:38:04,120 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:38:04,479 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:38:05,479 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:38:06,479 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:38:07,479 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:38:08,479 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:38:09,479 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:38:10,120 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:38:10,479 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:38:11,479 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:38:12,480 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:38:13,018 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 17:38:13,019 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 17:38:13,480 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:38:14,480 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:38:15,121 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:38:15,480 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:38:15,834 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 17:38:16,480 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:38:17,480 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:38:18,480 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:38:19,480 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:38:20,480 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:38:21,121 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:38:21,480 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:38:22,484 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:38:23,489 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:38:24,484 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:38:25,484 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:38:26,126 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:38:26,487 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:38:27,484 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:38:27,912 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 17:38:27,913 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 17:38:27,914 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 17:38:27,915 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 17:38:28,063 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 17:38:28,320 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 17:38:28,321 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 17:38:28,485 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:38:29,484 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:38:30,065 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 17:38:30,484 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:38:31,447 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:38:31,485 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:38:32,485 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:38:33,485 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:38:34,485 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:38:35,485 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:38:36,485 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:38:37,317 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:38:37,485 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:38:38,485 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:38:39,485 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:38:40,485 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:38:41,486 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:38:42,486 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:38:43,057 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 17:38:43,058 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 17:38:43,214 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:38:43,486 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:38:44,486 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:38:45,486 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:38:45,838 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 17:38:46,486 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:38:47,486 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:38:48,317 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:38:48,486 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:38:49,486 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:38:50,486 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:38:51,486 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:38:52,487 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:38:53,324 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:38:53,488 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:38:54,490 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:38:55,494 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:38:56,497 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:38:57,501 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:38:58,059 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 17:38:58,059 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 17:38:58,498 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:38:58,751 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 17:38:58,753 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 17:38:58,753 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 17:38:58,753 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:38:58,754 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 17:38:59,101 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 17:38:59,499 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:39:00,102 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 17:39:00,499 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:39:01,499 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:39:02,499 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:39:03,499 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:39:04,324 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:39:04,499 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:39:05,499 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:39:06,499 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:39:07,499 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:39:08,499 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:39:09,499 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:39:10,325 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:39:10,499 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:39:11,499 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:39:12,500 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:39:13,059 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 17:39:13,060 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 17:39:13,500 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:39:14,500 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:39:15,500 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:39:15,842 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 17:39:15,848 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:39:16,500 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:39:17,500 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:39:18,500 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:39:19,500 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:39:20,501 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:39:21,325 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:39:21,500 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:39:22,500 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:39:23,501 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:39:24,501 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:39:25,501 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:39:26,330 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:39:26,506 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:39:27,509 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:39:28,061 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 17:39:28,062 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 17:39:28,509 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:39:29,506 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:39:30,226 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 17:39:30,227 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 17:39:30,227 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 17:39:30,228 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 17:39:30,627 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:39:31,132 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 17:39:31,626 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:39:32,132 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 17:39:32,268 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:39:32,626 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:39:33,626 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:39:34,626 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:39:35,626 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:39:36,626 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:39:37,328 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:39:37,626 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:39:38,626 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:39:39,626 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:39:40,626 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:39:41,627 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:39:42,627 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:39:43,122 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 17:39:43,122 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 17:39:43,293 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:39:43,627 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:39:44,627 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:39:45,627 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:39:45,846 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 17:39:46,627 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:39:47,627 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:39:48,328 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:39:48,627 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:39:49,627 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:39:50,627 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:39:51,627 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:39:52,628 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:39:53,328 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:39:53,628 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:39:54,628 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:39:55,631 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:39:56,632 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:39:57,639 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:39:58,123 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 17:39:58,123 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 17:39:58,332 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:39:58,635 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:39:59,635 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:40:00,395 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 17:40:00,396 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 17:40:00,396 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 17:40:00,397 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 17:40:00,791 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:40:01,158 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 17:40:01,790 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:40:02,159 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 17:40:02,790 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:40:03,333 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:40:03,790 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:40:04,790 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:40:05,790 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:40:06,790 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:40:07,790 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:40:08,790 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:40:09,333 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:40:09,790 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:40:10,790 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:40:11,791 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:40:12,791 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:40:13,123 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 17:40:13,124 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 17:40:13,791 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:40:14,791 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:40:15,333 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:40:15,791 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:40:15,850 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 17:40:16,791 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:40:17,791 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:40:18,791 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:40:19,791 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:40:20,333 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:40:20,791 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:40:21,791 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:40:22,792 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:40:23,792 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:40:24,805 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:40:25,336 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:40:25,798 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:40:26,800 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:40:27,800 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:40:28,128 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 17:40:28,132 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 17:40:28,801 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:40:29,801 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:40:29,814 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 17:40:29,815 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 17:40:29,815 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 17:40:29,816 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 17:40:30,182 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 17:40:30,801 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:40:30,855 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:40:31,801 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:40:32,184 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 17:40:32,801 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:40:33,801 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:40:34,801 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:40:35,801 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:40:36,338 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:40:36,801 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:40:37,801 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:40:38,801 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:40:39,801 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:40:40,802 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:40:41,802 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:40:42,338 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:40:42,802 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:40:43,126 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 17:40:43,126 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 17:40:43,802 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:40:44,836 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:40:45,832 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:40:45,854 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 17:40:46,833 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:40:47,833 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:40:47,856 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:40:48,833 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:40:49,833 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:40:50,833 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:40:51,833 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:40:52,833 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:40:53,338 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:40:53,833 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:40:54,833 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:40:55,833 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:40:56,840 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:40:57,837 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:40:58,134 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 17:40:58,135 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 17:40:58,347 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:40:58,837 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:40:59,837 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:41:00,837 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:41:01,837 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:41:02,490 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 17:41:02,492 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 17:41:02,492 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 17:41:02,494 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 17:41:02,884 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:41:03,217 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 17:41:03,883 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:41:04,218 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 17:41:04,342 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:41:04,883 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:41:05,883 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:41:06,883 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:41:07,883 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:41:08,883 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:41:09,883 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:41:10,342 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:41:10,884 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:41:11,884 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:41:12,884 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:41:13,132 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 17:41:13,132 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 17:41:13,884 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:41:14,884 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:41:15,858 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 17:41:15,864 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:41:15,884 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:41:16,884 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:41:17,885 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:41:18,884 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:41:19,885 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:41:20,885 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:41:21,342 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:41:21,885 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:41:22,885 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:41:23,885 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:41:24,885 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:41:25,885 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:41:26,343 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:41:26,892 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:41:27,890 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:41:28,139 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 17:41:28,146 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 17:41:28,890 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:41:29,894 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:41:30,892 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:41:31,358 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:41:31,667 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 17:41:31,668 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 17:41:31,669 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 17:41:31,670 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 17:41:32,069 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:41:32,246 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 17:41:33,063 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:41:34,065 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:41:34,248 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 17:41:35,063 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:41:36,063 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:41:37,063 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:41:37,347 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:41:38,063 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:41:39,064 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:41:40,064 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:41:41,064 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:41:42,064 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:41:43,064 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:41:43,136 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 17:41:43,136 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 17:41:43,199 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:41:44,064 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:41:45,064 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:41:45,862 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 17:41:46,065 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:41:47,064 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:41:48,065 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:41:48,347 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:41:49,064 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:41:50,065 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:41:51,065 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:41:52,065 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:41:53,065 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:41:54,065 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:41:54,347 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:41:55,065 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:41:56,065 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:41:57,065 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:41:58,065 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:41:58,136 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 17:41:58,137 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 17:41:59,073 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:41:59,367 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:42:00,083 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:42:01,077 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:42:02,079 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:42:03,077 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:42:03,986 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 17:42:03,988 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 17:42:03,988 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 17:42:03,989 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 17:42:04,276 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 17:42:04,381 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:42:05,027 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:42:05,131 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:42:06,131 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:42:06,278 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 17:42:07,131 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:42:08,131 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:42:09,131 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:42:10,131 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:42:10,354 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:42:11,131 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:42:12,131 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:42:13,131 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:42:13,136 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 17:42:13,136 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 17:42:14,131 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:42:15,132 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:42:15,866 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 17:42:15,873 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:42:16,132 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:42:17,132 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:42:18,132 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:42:19,132 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:42:20,132 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:42:21,132 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:42:21,354 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:42:22,132 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:42:23,132 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:42:24,132 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:42:25,132 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:42:26,133 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:42:26,376 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:42:27,139 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:42:28,141 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:42:28,142 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 17:42:28,142 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 17:42:29,141 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:42:30,143 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:42:31,143 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:42:32,071 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 17:42:32,073 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 17:42:32,073 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 17:42:32,073 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:42:32,074 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 17:42:32,301 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 17:42:32,448 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:42:33,448 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:42:34,303 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 17:42:34,449 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:42:35,449 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:42:36,449 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:42:37,355 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:42:37,449 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:42:38,449 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:42:39,449 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:42:40,449 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:42:41,449 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:42:42,450 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:42:43,140 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 17:42:43,140 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 17:42:43,227 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:42:43,450 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:42:44,450 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:42:45,450 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:42:45,870 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 17:42:46,450 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:42:47,450 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:42:48,354 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:42:48,450 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:42:49,450 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:42:50,450 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:42:51,450 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:42:52,451 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:42:53,355 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:42:53,451 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:42:54,451 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:42:55,451 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:42:56,451 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:42:57,460 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:42:58,142 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 17:42:58,143 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 17:42:58,456 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:42:59,356 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:42:59,460 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:43:00,460 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:43:01,461 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:43:02,461 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:43:03,264 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 17:43:03,266 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 17:43:03,266 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 17:43:03,267 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 17:43:03,332 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 17:43:03,640 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:43:04,333 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 17:43:04,636 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:43:04,637 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:43:05,636 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:43:06,636 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:43:07,636 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:43:08,636 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:43:09,636 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:43:10,410 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:43:10,636 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:43:11,636 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:43:12,636 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:43:13,141 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 17:43:13,141 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 17:43:13,636 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:43:14,637 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:43:15,637 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:43:15,876 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 17:43:15,882 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:43:16,638 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:43:17,637 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:43:18,637 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:43:19,638 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:43:20,638 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:43:21,411 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:43:21,638 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:43:22,638 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:43:23,638 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:43:24,638 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:43:25,638 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:43:26,639 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:43:27,413 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:43:27,647 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:43:28,147 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 17:43:28,147 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 17:43:28,649 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:43:29,647 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:43:30,649 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:43:31,649 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:43:32,415 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:43:32,649 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:43:32,681 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 17:43:32,682 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 17:43:32,683 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 17:43:32,685 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 17:43:33,361 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 17:43:33,649 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:43:34,362 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 17:43:34,649 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:43:35,649 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:43:36,649 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:43:37,650 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:43:38,414 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:43:38,650 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:43:39,650 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:43:40,650 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:43:41,650 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:43:42,650 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:43:43,147 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 17:43:43,147 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 17:43:43,650 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:43:44,414 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:43:44,650 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:43:45,651 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:43:45,878 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 17:43:46,650 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:43:47,650 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:43:48,650 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:43:49,414 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:43:49,651 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:43:50,651 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:43:51,651 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:43:52,651 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:43:53,651 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:43:54,651 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:43:55,414 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:43:55,651 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:43:56,651 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:43:57,655 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:43:58,152 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 17:43:58,153 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 17:43:58,660 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:43:59,660 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:44:00,421 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:44:00,664 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:44:01,660 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:44:02,660 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:44:02,885 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 17:44:02,886 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 17:44:02,886 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 17:44:02,887 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 17:44:03,389 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 17:44:03,660 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:44:04,390 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 17:44:04,660 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:44:05,660 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:44:06,415 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:44:06,661 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:44:07,661 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:44:08,661 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:44:09,661 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:44:10,661 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:44:11,661 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:44:12,415 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:44:12,661 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:44:13,163 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 17:44:13,164 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 17:44:13,661 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:44:14,661 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:44:15,661 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:44:15,882 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 17:44:16,662 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:44:17,662 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:44:17,885 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:44:18,662 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:44:19,662 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:44:20,662 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:44:21,662 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:44:22,662 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:44:23,414 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:44:23,670 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:44:24,666 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:44:25,666 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:44:26,666 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:44:27,666 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:44:28,162 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 17:44:28,162 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 17:44:28,415 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:44:28,508 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 17:44:28,510 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 17:44:28,510 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 17:44:28,512 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 17:44:28,909 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:44:29,419 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 17:44:29,908 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:44:30,420 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 17:44:30,908 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:44:31,908 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:44:32,908 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:44:33,908 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:44:34,415 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:44:34,908 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:44:35,909 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:44:36,909 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:44:37,909 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:44:38,909 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:44:39,975 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:44:40,415 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:44:40,970 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:44:41,970 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:44:42,970 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:44:43,162 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 17:44:43,162 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 17:44:43,971 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:44:44,971 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:44:45,886 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 17:44:45,892 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:44:45,971 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:44:46,971 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:44:47,972 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:44:48,971 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:44:49,971 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:44:50,971 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:44:51,425 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:44:51,974 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:44:52,976 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:44:53,978 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:44:54,978 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:44:55,978 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:44:56,428 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:44:56,978 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:44:56,984 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 17:44:56,985 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 17:44:56,985 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 17:44:56,986 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 17:44:57,448 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 17:44:57,978 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:44:58,162 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 17:44:58,163 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 17:44:58,449 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 17:44:58,978 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:44:59,978 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:45:00,979 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:45:01,979 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:45:02,422 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:45:02,979 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:45:03,979 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:45:04,979 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:45:05,979 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:45:06,979 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:45:07,979 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:45:08,422 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:45:08,979 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:45:09,979 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:45:11,373 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:45:12,109 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:45:13,109 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:45:13,162 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 17:45:13,163 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 17:45:14,109 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:45:14,422 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:45:15,109 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:45:15,890 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 17:45:16,109 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:45:17,109 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:45:18,109 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:45:19,109 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:45:20,109 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:45:20,422 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:45:21,109 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:45:22,110 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:45:23,110 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:45:24,120 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:45:25,115 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:45:25,427 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:45:26,116 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:45:27,116 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:45:28,118 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:45:28,170 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 17:45:28,173 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 17:45:29,116 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 17:45:29,117 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 17:45:29,117 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 17:45:29,119 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 17:45:29,157 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:45:29,475 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 17:45:30,125 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:45:30,476 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 17:45:31,124 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:45:31,158 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:45:32,124 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:45:33,124 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:45:34,124 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:45:35,124 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:45:36,124 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:45:36,425 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:45:37,124 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:45:38,124 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:45:39,124 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:45:40,125 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:45:41,125 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:45:42,125 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:45:42,425 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:45:43,125 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:45:43,170 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 17:45:43,170 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 17:45:44,125 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:45:45,125 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:45:45,895 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 17:45:46,125 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:45:47,125 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:45:47,897 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:45:48,125 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:45:49,125 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:45:50,127 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:45:51,126 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:45:52,126 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:45:53,126 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:45:53,432 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:45:54,131 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:45:55,132 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:45:56,140 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:45:57,138 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:45:58,136 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:45:58,170 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 17:45:58,170 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 17:45:58,431 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:45:59,136 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:45:59,163 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 17:45:59,164 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 17:45:59,165 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 17:45:59,166 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 17:45:59,502 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 17:46:00,136 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:46:00,503 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 17:46:01,136 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:46:02,136 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:46:03,136 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:46:03,432 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:46:04,137 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:46:05,137 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:46:06,137 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:46:07,137 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:46:08,137 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:46:09,137 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:46:09,432 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:46:10,137 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:46:11,137 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:46:12,138 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:46:13,137 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:46:13,170 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 17:46:13,171 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 17:46:14,138 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:46:15,138 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:46:15,432 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:46:15,898 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 17:46:16,138 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:46:17,138 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:46:18,138 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:46:19,138 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:46:20,138 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:46:20,432 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:46:21,138 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:46:22,138 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:46:23,138 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:46:24,149 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:46:25,146 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:46:25,438 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:46:26,147 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:46:27,149 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:46:28,145 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:46:28,175 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 17:46:28,176 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 17:46:29,146 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:46:29,381 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 17:46:29,382 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 17:46:29,382 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 17:46:29,384 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 17:46:29,526 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 17:46:30,146 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:46:30,527 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 17:46:31,149 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:46:31,422 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:46:32,148 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:46:33,149 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:46:34,149 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:46:35,149 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:46:36,149 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:46:36,432 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:46:37,149 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:46:38,149 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:46:39,149 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:46:40,149 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:46:41,149 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:46:42,223 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:46:42,433 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:46:43,171 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 17:46:43,172 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 17:46:43,222 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:46:44,222 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:46:45,222 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:46:45,902 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 17:46:46,222 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:46:47,222 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:46:47,905 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:46:48,222 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:46:49,223 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:46:50,223 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:46:51,223 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:46:52,223 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:46:53,223 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:46:53,433 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:46:54,228 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:46:55,234 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:46:56,236 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:46:57,239 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:46:58,175 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 17:46:58,175 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 17:46:58,238 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:46:58,442 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:46:59,236 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:47:00,027 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 17:47:00,028 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 17:47:00,029 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 17:47:00,030 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 17:47:00,429 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:47:00,558 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 17:47:01,428 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:47:02,428 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:47:02,559 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 17:47:03,428 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:47:04,428 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:47:04,439 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:47:05,428 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:47:06,428 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:47:07,428 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:47:08,428 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:47:09,428 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:47:10,429 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:47:10,439 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:47:11,809 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:47:12,490 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:47:13,175 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 17:47:13,176 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 17:47:13,490 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:47:14,490 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:47:15,490 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:47:15,906 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 17:47:15,913 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:47:16,490 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:47:17,490 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:47:18,490 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:47:19,490 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:47:20,490 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:47:21,439 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:47:21,491 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:47:22,496 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:47:23,496 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:47:24,503 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:47:25,499 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:47:26,444 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:47:26,499 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:47:27,499 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:47:27,607 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 17:47:27,608 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 17:47:27,608 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 17:47:27,610 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 17:47:28,175 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 17:47:28,176 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 17:47:28,499 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:47:28,582 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 17:47:28,583 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 17:47:29,499 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:47:30,499 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:47:31,499 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:47:32,441 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:47:32,499 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:47:33,500 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:47:34,500 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:47:35,500 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:47:36,500 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:47:37,441 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:47:37,500 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:47:38,500 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:47:39,500 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:47:40,795 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:47:41,794 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:47:42,794 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:47:43,175 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 17:47:43,176 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 17:47:43,263 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:47:43,794 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:47:44,794 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:47:45,794 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:47:45,910 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 17:47:46,794 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:47:47,794 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:47:48,441 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:47:48,794 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:47:49,795 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:47:50,799 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:47:51,803 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:47:52,802 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:47:53,447 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:47:53,803 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:47:54,802 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:47:55,802 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:47:56,271 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 17:47:56,272 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 17:47:56,273 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 17:47:56,274 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 17:47:56,604 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 17:47:56,802 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:47:57,802 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:47:58,175 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 17:47:58,176 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 17:47:58,606 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 17:47:58,803 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:47:59,445 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:47:59,802 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:48:00,802 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:48:01,803 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:48:02,803 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:48:03,803 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:48:04,803 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:48:05,445 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:48:05,803 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:48:06,803 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:48:07,866 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:48:08,863 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:48:09,863 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:48:10,445 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:48:10,864 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:48:11,864 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:48:12,864 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:48:13,175 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 17:48:13,176 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 17:48:13,864 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:48:14,865 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:48:15,864 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:48:15,915 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 17:48:15,919 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:48:16,864 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:48:17,864 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:48:18,864 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:48:19,871 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-25 17:48:38,855 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:48:38,855 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 17:48:38,856 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:48:38,857 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 17:48:38,857 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:48:38,857 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 17:48:38,857 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:48:38,857 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 17:48:38,858 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 17:48:38,859 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 17:48:39,203 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 17:48:39,203 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 17:48:40,162 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 17:48:42,311 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:48:43,176 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 17:48:43,176 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 17:48:45,917 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 17:48:47,562 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:48:48,636 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 17:48:52,570 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:48:53,646 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 17:48:57,571 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:48:57,758 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 17:48:57,760 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 17:48:57,760 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 17:48:57,761 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 17:48:58,176 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 17:48:58,177 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 17:49:03,329 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:49:03,644 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 17:49:08,330 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:49:08,645 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 17:49:13,177 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 17:49:13,177 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 17:49:14,312 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:49:15,921 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 17:49:18,646 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 17:49:19,923 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:49:23,652 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 17:49:24,935 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:49:27,012 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 17:49:27,013 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 17:49:27,014 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 17:49:27,019 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 17:49:28,176 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 17:49:28,177 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 17:49:30,306 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:49:33,653 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 17:49:35,307 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:49:38,654 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 17:49:40,307 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:49:43,176 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 17:49:43,177 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 17:49:45,322 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:49:45,925 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 17:49:48,407 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 17:49:50,928 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:49:53,414 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 17:49:54,502 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 17:49:55,503 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 17:49:55,503 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 17:49:55,935 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:49:56,507 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 17:49:58,178 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 17:49:58,178 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 17:49:58,909 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 17:49:58,910 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 17:49:58,910 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 17:49:58,912 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 17:49:59,512 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 17:50:00,513 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 17:50:01,913 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:50:03,416 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 17:50:06,914 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:50:08,417 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 17:50:11,914 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:50:13,178 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 17:50:13,179 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 17:50:15,929 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 17:50:16,931 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:50:18,418 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 17:50:21,931 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:50:23,424 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 17:50:26,938 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:50:28,180 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 17:50:28,180 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 17:50:28,609 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 17:50:28,611 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 17:50:28,611 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 17:50:28,613 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 17:50:29,541 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 17:50:31,543 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 17:50:32,614 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:50:33,431 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 17:50:37,614 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:50:38,432 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 17:50:42,615 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:50:43,180 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 17:50:43,181 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 17:50:45,933 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 17:50:47,935 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:50:48,433 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 17:50:52,942 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:50:53,441 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 17:50:57,534 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 17:50:57,536 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 17:50:57,536 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 17:50:57,537 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 17:50:57,567 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 17:50:58,180 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 17:50:58,181 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 17:50:58,302 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:50:59,568 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 17:51:03,302 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:51:03,440 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 17:51:08,303 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:51:08,441 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 17:51:13,180 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 17:51:13,181 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 17:51:14,296 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:51:15,938 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 17:51:18,442 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 17:51:19,940 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:51:23,449 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 17:51:24,946 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:51:28,187 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 17:51:28,187 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 17:51:28,804 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 17:51:28,805 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 17:51:28,805 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 17:51:28,806 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 17:51:29,602 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 17:51:30,807 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:51:31,604 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 17:51:33,202 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 17:51:35,807 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:51:38,203 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 17:51:40,808 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:51:43,187 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 17:51:43,188 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 17:51:45,941 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 17:51:45,947 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:51:48,204 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 17:51:50,944 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:51:53,205 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 17:51:55,955 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:51:58,194 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 17:51:58,202 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 17:51:59,434 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 17:51:59,435 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 17:51:59,436 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 17:51:59,438 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 17:51:59,629 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 17:52:01,439 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:52:01,631 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 17:52:03,214 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 17:52:06,440 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:52:08,214 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 17:52:11,440 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:52:13,191 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 17:52:13,192 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 17:52:15,945 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 17:52:16,947 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:52:18,245 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 17:52:21,948 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:52:23,254 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 17:52:26,955 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:52:28,194 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 17:52:28,194 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 17:52:28,669 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 17:52:28,670 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 17:52:28,670 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 17:52:28,671 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 17:52:29,656 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 17:52:31,658 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 17:52:32,673 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:52:33,251 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 17:52:37,673 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:52:38,252 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 17:52:42,674 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:52:43,194 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 17:52:43,195 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 17:52:45,949 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 17:52:47,952 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:52:48,253 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 17:52:52,952 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:52:53,266 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 17:52:57,959 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:52:58,198 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 17:52:58,198 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 17:52:58,955 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 17:52:58,956 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 17:52:58,957 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 17:52:58,959 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 17:52:59,683 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 17:53:01,685 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 17:53:02,960 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:53:03,372 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 17:53:07,961 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:53:08,373 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 17:53:12,961 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:53:13,198 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 17:53:13,198 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 17:53:15,953 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 17:53:18,374 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 17:53:18,956 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:53:23,383 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 17:53:23,964 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:53:28,202 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 17:53:28,202 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 17:53:28,941 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 17:53:28,942 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 17:53:28,942 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 17:53:28,944 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 17:53:29,708 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 17:53:29,945 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:53:31,709 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 17:53:33,382 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 17:53:34,945 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:53:38,382 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 17:53:39,946 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:53:43,202 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 17:53:43,203 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 17:53:45,314 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:53:45,958 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 17:53:48,423 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 17:53:50,961 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:53:53,437 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 17:53:55,974 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:53:57,328 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 17:53:57,330 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 17:53:57,330 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 17:53:57,332 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 17:53:57,730 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 17:53:58,202 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 17:53:58,203 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 17:53:59,732 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 17:54:01,289 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:54:03,475 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 17:54:06,290 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:54:08,476 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 17:54:11,292 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:54:13,202 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 17:54:13,203 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 17:54:15,961 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 17:54:16,963 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:54:18,477 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 17:54:21,969 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:54:23,488 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 17:54:24,859 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 17:54:24,861 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 17:54:24,861 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 17:54:24,862 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 17:54:25,753 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 17:54:27,755 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 17:54:27,863 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:54:28,202 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 17:54:28,203 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 17:54:33,378 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:54:33,484 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 17:54:38,379 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:54:38,485 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 17:54:43,203 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 17:54:43,203 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 17:54:44,319 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:54:45,965 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 17:54:48,494 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 17:54:49,974 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:54:53,494 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 17:54:54,217 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 17:54:54,219 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 17:54:54,219 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 17:54:54,221 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 17:54:54,783 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 17:54:55,222 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:54:55,783 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 17:54:58,203 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 17:54:58,203 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 17:55:00,352 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:55:03,674 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 17:55:05,353 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:55:08,674 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 17:55:10,354 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:55:13,203 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 17:55:13,203 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 17:55:15,372 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:55:15,969 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 17:55:18,675 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 17:55:20,972 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:55:23,684 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 17:55:25,981 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:55:27,538 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 17:55:27,540 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 17:55:27,540 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 17:55:27,542 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 17:55:27,811 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 17:55:28,203 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 17:55:28,204 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 17:55:29,813 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 17:55:31,374 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:55:33,685 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 17:55:36,375 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:55:38,686 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 17:55:41,375 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:55:43,203 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 17:55:43,204 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 17:55:45,973 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 17:55:46,975 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:55:48,687 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 17:55:51,981 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:55:53,697 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 17:55:56,476 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 17:55:56,477 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 17:55:56,477 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 17:55:56,479 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 17:55:56,837 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 17:55:57,479 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:55:57,838 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 17:55:58,203 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 17:55:58,204 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 17:56:03,295 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:56:03,865 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 17:56:08,296 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:56:08,866 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 17:56:13,203 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 17:56:13,204 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 17:56:13,340 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:56:15,977 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 17:56:18,867 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 17:56:18,979 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:56:23,876 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 17:56:23,991 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:56:26,134 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 17:56:26,136 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 17:56:26,137 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 17:56:26,138 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 17:56:26,864 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 17:56:27,865 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 17:56:28,206 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 17:56:28,207 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 17:56:29,376 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:56:33,876 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 17:56:34,376 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:56:38,877 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 17:56:39,377 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:56:43,206 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 17:56:43,207 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 17:56:45,362 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:56:45,981 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 17:56:48,879 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 17:56:50,995 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:56:53,887 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 17:56:54,704 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 17:56:54,705 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 17:56:54,706 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 17:56:54,707 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 17:56:54,890 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 17:56:55,892 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 17:56:56,708 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:56:58,206 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 17:56:58,207 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 17:57:02,345 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:57:04,107 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 17:57:04,108 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 17:57:07,346 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:57:09,108 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 17:57:09,109 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 17:57:12,346 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:57:13,206 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 17:57:13,207 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 17:57:15,985 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 17:57:17,987 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:57:19,112 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 17:57:22,996 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:57:23,909 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 17:57:23,910 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 17:57:23,910 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 17:57:23,911 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 17:57:23,922 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 17:57:24,311 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 17:57:24,311 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 17:57:25,924 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 17:57:28,206 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 17:57:28,207 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 17:57:28,291 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:57:33,291 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:57:33,312 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 17:57:38,292 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:57:38,313 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 17:57:43,206 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 17:57:43,207 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 17:57:43,295 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:57:45,989 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 17:57:48,323 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 17:57:48,324 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 17:57:48,999 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:57:52,585 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 17:57:52,586 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 17:57:52,587 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 17:57:52,588 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 17:57:52,954 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 17:57:53,323 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 17:57:53,323 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 17:57:53,955 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 17:57:54,589 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:57:58,206 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 17:57:58,207 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 17:58:00,354 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:58:03,324 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 17:58:03,324 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 17:58:05,354 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:58:08,325 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 17:58:08,325 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 17:58:10,355 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:58:13,207 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 17:58:13,207 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 17:58:15,359 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:58:15,993 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 17:58:18,330 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 17:58:20,999 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:58:22,285 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 17:58:22,287 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 17:58:22,287 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 17:58:22,289 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 17:58:22,979 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 17:58:23,430 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 17:58:23,979 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 17:58:26,290 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:58:28,207 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 17:58:28,207 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 17:58:31,311 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:58:33,431 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 17:58:36,312 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:58:38,540 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 17:58:41,312 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:58:43,207 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 17:58:43,207 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 17:58:45,997 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 17:58:46,999 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:58:48,541 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 17:58:52,003 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:58:53,552 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 17:58:54,700 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 17:58:54,701 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 17:58:54,702 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 17:58:54,703 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 17:58:55,013 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 17:58:56,014 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 17:58:57,704 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:58:58,207 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 17:58:58,207 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 17:59:03,321 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:59:03,546 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 17:59:08,321 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:59:08,623 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 17:59:13,207 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 17:59:13,208 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 17:59:13,368 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:59:16,001 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 17:59:18,624 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 17:59:19,003 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:59:23,637 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 17:59:24,007 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:59:25,308 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 17:59:25,310 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 17:59:25,310 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 17:59:25,312 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 17:59:26,038 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 17:59:28,041 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 17:59:28,207 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 17:59:28,208 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 17:59:29,365 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:59:33,708 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 17:59:34,365 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:59:38,709 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 17:59:39,366 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:59:43,207 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 17:59:43,208 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 17:59:45,299 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:59:46,004 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 17:59:48,710 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 17:59:51,010 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:59:53,723 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 17:59:55,680 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 17:59:55,682 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 17:59:55,683 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 17:59:55,684 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 17:59:56,067 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 17:59:56,684 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 17:59:58,068 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 17:59:58,207 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 17:59:58,208 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 18:00:02,288 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:00:03,826 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:00:07,289 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:00:08,827 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:00:12,289 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:00:13,208 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 18:00:13,208 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 18:00:16,008 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 18:00:18,010 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:00:18,828 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:00:23,017 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:00:23,831 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:00:28,020 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:00:28,208 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 18:00:28,208 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 18:00:28,369 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 18:00:28,371 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 18:00:28,372 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 18:00:28,373 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 18:00:29,094 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 18:00:30,095 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 18:00:33,374 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:00:33,836 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:00:38,375 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:00:38,837 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:00:43,208 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 18:00:43,209 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 18:00:44,337 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:00:46,012 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 18:00:48,839 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:00:50,014 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:00:53,848 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:00:55,023 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:00:56,463 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 18:00:56,465 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 18:00:56,465 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 18:00:56,466 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 18:00:57,122 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 18:00:58,122 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 18:00:58,208 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 18:00:58,209 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 18:01:00,348 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:01:03,857 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:01:05,349 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:01:08,858 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:01:10,350 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:01:13,208 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 18:01:13,209 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 18:01:15,396 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:01:16,016 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 18:01:18,871 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:01:21,026 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:01:23,675 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 18:01:23,677 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 18:01:23,677 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 18:01:23,678 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 18:01:24,071 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:01:24,150 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 18:01:26,151 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 18:01:26,679 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:01:28,208 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 18:01:28,209 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 18:01:32,286 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:01:34,077 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:01:37,287 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:01:39,078 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:01:42,288 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:01:43,208 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 18:01:43,209 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 18:01:46,020 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 18:01:48,023 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:01:49,079 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:01:53,032 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:01:54,091 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:01:57,431 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 18:01:57,432 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 18:01:57,433 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 18:01:57,434 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 18:01:58,182 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 18:01:58,209 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 18:01:58,209 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 18:01:58,382 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:02:00,185 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 18:02:03,382 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:02:04,085 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:02:08,383 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:02:09,091 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:02:13,209 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 18:02:13,209 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 18:02:14,308 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:02:16,024 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 18:02:19,093 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:02:20,026 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:02:24,103 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:02:25,033 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:02:26,558 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 18:02:26,560 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 18:02:26,560 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 18:02:26,562 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 18:02:27,212 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 18:02:28,209 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 18:02:28,209 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 18:02:28,216 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 18:02:30,347 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:02:34,105 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:02:35,347 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:02:39,105 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:02:40,348 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:02:43,209 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 18:02:43,209 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 18:02:45,489 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:02:46,028 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 18:02:48,255 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:02:51,030 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:02:53,256 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:02:53,256 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:02:56,032 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:02:58,214 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 18:02:58,221 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 18:03:00,897 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 18:03:00,899 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 18:03:00,899 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 18:03:00,901 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 18:03:01,251 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 18:03:01,902 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:03:02,252 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 18:03:03,302 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:03:06,902 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:03:08,302 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:03:11,903 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:03:13,211 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 18:03:13,212 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 18:03:16,032 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 18:03:17,034 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:03:18,339 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:03:18,339 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:03:22,035 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:03:23,340 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:03:23,340 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:03:27,042 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:03:28,221 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 18:03:28,222 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 18:03:29,569 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 18:03:29,570 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 18:03:29,571 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 18:03:29,572 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 18:03:30,280 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 18:03:32,281 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 18:03:32,577 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:03:33,349 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:03:37,577 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:03:38,350 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:03:42,578 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:03:43,214 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 18:03:43,215 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 18:03:46,036 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 18:03:48,038 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:03:48,351 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:03:53,044 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:03:53,357 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:03:58,048 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:03:58,166 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 18:03:58,168 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 18:03:58,168 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 18:03:58,170 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 18:03:58,310 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 18:03:58,557 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:03:58,561 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 18:03:58,562 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 18:04:00,311 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 18:04:03,725 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:04:04,558 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:04:08,726 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:04:09,559 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:04:13,312 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 18:04:13,313 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 18:04:14,483 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:04:16,040 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 18:04:18,560 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:04:20,043 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:04:23,565 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:04:25,047 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:04:27,054 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 18:04:27,056 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 18:04:27,056 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 18:04:27,058 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 18:04:27,337 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 18:04:28,312 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 18:04:28,313 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 18:04:28,338 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 18:04:30,463 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:04:33,567 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:04:35,464 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:04:38,567 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:04:40,465 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:04:43,312 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 18:04:43,313 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 18:04:46,045 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 18:04:46,051 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:04:48,576 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:04:51,059 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:04:53,573 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:04:53,968 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 18:04:53,970 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 18:04:53,970 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 18:04:53,971 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 18:04:54,363 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 18:04:56,365 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 18:04:56,972 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:04:58,312 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 18:04:58,313 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 18:05:02,440 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:05:03,578 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:05:07,441 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:05:08,631 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:05:08,631 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:05:12,442 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:05:13,313 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 18:05:13,313 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 18:05:16,048 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 18:05:18,050 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:05:18,645 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:05:23,058 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:05:23,638 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:05:24,136 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 18:05:24,138 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 18:05:24,138 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 18:05:24,140 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 18:05:24,389 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 18:05:26,390 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 18:05:28,141 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:05:28,313 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 18:05:28,313 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 18:05:33,463 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:05:33,640 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:05:38,464 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:05:38,641 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:05:43,313 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 18:05:43,313 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 18:05:44,435 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:05:46,052 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 18:05:48,647 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:05:50,060 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:05:53,322 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 18:05:53,324 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 18:05:53,324 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 18:05:53,326 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 18:05:53,414 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 18:05:53,721 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:05:53,721 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:05:54,416 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 18:05:55,327 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:05:58,313 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 18:05:58,314 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 18:06:00,472 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:06:03,726 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:06:03,726 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:06:05,473 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:06:08,736 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:06:08,736 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:06:10,474 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:06:13,313 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 18:06:13,313 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 18:06:16,056 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 18:06:16,062 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:06:18,746 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:06:21,063 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:06:23,743 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:06:24,120 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 18:06:24,121 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 18:06:24,121 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 18:06:24,123 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 18:06:24,444 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 18:06:24,445 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 18:06:26,124 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:06:28,313 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 18:06:28,314 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 18:06:31,480 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:06:33,745 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:06:36,481 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:06:38,746 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:06:41,481 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:06:43,313 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 18:06:43,314 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 18:06:46,060 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 18:06:47,062 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:06:48,747 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:06:52,067 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:06:53,754 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:06:54,937 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 18:06:54,939 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 18:06:54,940 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 18:06:54,941 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 18:06:55,481 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 18:06:56,482 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 18:06:57,942 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:06:58,313 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 18:06:58,314 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 18:07:03,393 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:07:03,753 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:07:08,393 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:07:08,753 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:07:13,314 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 18:07:13,314 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 18:07:13,485 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:07:16,065 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 18:07:18,755 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:07:19,067 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:07:23,760 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:07:24,072 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:07:26,525 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 18:07:26,526 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 18:07:26,527 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 18:07:26,528 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 18:07:27,507 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 18:07:28,314 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 18:07:28,314 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 18:07:28,508 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 18:07:29,447 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:07:33,918 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:07:34,447 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:07:39,051 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:07:39,051 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:07:39,448 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:07:43,314 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 18:07:43,314 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 18:07:45,419 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:07:46,068 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 18:07:49,053 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:07:49,053 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:07:51,075 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:07:54,058 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:07:55,385 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 18:07:55,386 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 18:07:55,387 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 18:07:55,387 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 18:07:55,539 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 18:07:56,388 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:07:56,540 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 18:07:58,314 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 18:07:58,314 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 18:08:01,401 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:08:04,059 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:08:06,402 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:08:09,060 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:08:11,402 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:08:13,314 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 18:08:13,314 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 18:08:16,072 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 18:08:17,074 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:08:19,061 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:08:22,080 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:08:24,067 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:08:27,003 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 18:08:27,005 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 18:08:27,005 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 18:08:27,007 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 18:08:27,580 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 18:08:28,008 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:08:28,314 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 18:08:28,315 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 18:08:28,581 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 18:08:33,481 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:08:34,151 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:08:38,482 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:08:39,152 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:08:43,314 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 18:08:43,315 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 18:08:44,425 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:08:46,076 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 18:08:49,157 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:08:50,086 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:08:54,162 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:08:54,435 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 18:08:54,436 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 18:08:54,437 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 18:08:54,439 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 18:08:54,607 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 18:08:55,440 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:08:56,609 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 18:08:58,314 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 18:08:58,315 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 18:09:01,406 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:09:04,164 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:09:06,407 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:09:09,165 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:09:11,408 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:09:13,314 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 18:09:13,315 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 18:09:16,079 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 18:09:17,081 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:09:19,166 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:09:22,085 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:09:24,178 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:09:25,059 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 18:09:25,060 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 18:09:25,060 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 18:09:25,061 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 18:09:25,635 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 18:09:26,636 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 18:09:28,062 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:09:28,314 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 18:09:28,315 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 18:09:33,481 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:09:34,204 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:09:34,205 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:09:38,481 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:09:39,205 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:09:39,206 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:09:43,315 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 18:09:43,315 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 18:09:43,485 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:09:46,083 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 18:09:49,085 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:09:49,207 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:09:49,207 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:09:54,091 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:09:54,213 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:09:55,155 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 18:09:55,156 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 18:09:55,157 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 18:09:55,158 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 18:09:55,659 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 18:09:56,660 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 18:09:58,315 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 18:09:58,315 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 18:09:59,455 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:10:03,560 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:10:04,456 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:10:08,560 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:10:08,560 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:10:09,457 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:10:13,315 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 18:10:13,315 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 18:10:15,448 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:10:16,088 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 18:10:18,562 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:10:21,099 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:10:23,567 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:10:26,098 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:10:26,219 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 18:10:26,221 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 18:10:26,221 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 18:10:26,223 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 18:10:26,689 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 18:10:26,690 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 18:10:28,315 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 18:10:28,316 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 18:10:31,400 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:10:33,625 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:10:36,401 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:10:38,626 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:10:41,401 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:10:43,315 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 18:10:43,316 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 18:10:46,091 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 18:10:47,093 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:10:48,627 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:10:52,094 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:10:53,635 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:10:57,101 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:10:58,092 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 18:10:58,093 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 18:10:58,093 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 18:10:58,094 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 18:10:58,556 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 18:10:58,557 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 18:10:58,719 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 18:10:58,719 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 18:11:02,717 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:11:03,637 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:11:07,718 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:11:08,638 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:11:12,718 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:11:13,555 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 18:11:13,556 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 18:11:16,095 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 18:11:18,097 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:11:18,640 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:11:23,101 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:11:23,648 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:11:28,103 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:11:28,555 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 18:11:28,556 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 18:11:28,820 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 18:11:28,822 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 18:11:28,822 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 18:11:28,824 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 18:11:29,739 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 18:11:30,740 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 18:11:33,649 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:11:33,825 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:11:38,649 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:11:38,826 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:11:43,555 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 18:11:43,556 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 18:11:44,666 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:11:46,099 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 18:11:48,875 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:11:48,875 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:11:50,101 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:11:53,876 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:11:55,105 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:11:58,570 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 18:11:58,571 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 18:12:00,262 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 18:12:00,264 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 18:12:00,264 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 18:12:00,264 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:12:00,265 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 18:12:00,768 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 18:12:00,769 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 18:12:03,884 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:12:05,266 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:12:08,885 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:12:10,267 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:12:13,561 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 18:12:13,562 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 18:12:15,723 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:12:16,102 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 18:12:19,046 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:12:19,046 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:12:21,105 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:12:24,046 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:12:24,047 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:12:26,107 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:12:28,569 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 18:12:28,570 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 18:12:31,096 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 18:12:31,099 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 18:12:31,099 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 18:12:31,101 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 18:12:31,795 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 18:12:32,102 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:12:32,796 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 18:12:34,055 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:12:37,102 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:12:39,055 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:12:42,103 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:12:43,565 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 18:12:43,566 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 18:12:46,107 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 18:12:47,109 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:12:49,057 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:12:52,110 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:12:54,061 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:12:57,130 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:12:58,027 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 18:12:58,029 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 18:12:58,029 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 18:12:58,031 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 18:12:58,565 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 18:12:58,566 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 18:12:58,819 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 18:12:58,819 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 18:13:02,700 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:13:04,434 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:13:04,435 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:13:07,701 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:13:09,436 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:13:09,436 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:13:12,702 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:13:13,565 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 18:13:13,566 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 18:13:16,110 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 18:13:18,113 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:13:19,437 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:13:19,438 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:13:23,121 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:13:24,440 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:13:28,125 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:13:28,565 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 18:13:28,566 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 18:13:28,648 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 18:13:28,698 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 18:13:28,699 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 18:13:28,700 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 18:13:28,842 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 18:13:29,843 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 18:13:33,702 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:13:34,442 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:13:34,539 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:13:38,702 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:13:39,539 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:13:39,540 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:13:43,566 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 18:13:43,566 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 18:13:43,730 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:13:46,115 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 18:13:48,778 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:13:48,778 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:13:49,117 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:13:53,784 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:13:54,124 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:13:58,333 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 18:13:58,334 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 18:13:58,334 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 18:13:58,335 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 18:13:58,822 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:13:58,822 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:13:58,823 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 18:13:58,824 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 18:13:58,867 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 18:13:59,868 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 18:13:59,998 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:14:04,823 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:14:04,824 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:14:04,999 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:14:09,824 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:14:09,825 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:14:09,999 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:14:13,822 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 18:14:13,823 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 18:14:15,987 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:14:16,118 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 18:14:19,008 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:14:21,121 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:14:24,013 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:14:26,129 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:14:28,822 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 18:14:28,823 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 18:14:29,051 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 18:14:29,053 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 18:14:29,053 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 18:14:29,054 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 18:14:29,894 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 18:14:31,896 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 18:14:32,055 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:14:34,015 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:14:37,056 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:14:39,016 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:14:42,056 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:14:43,822 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 18:14:43,823 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 18:14:46,123 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 18:14:47,125 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:14:49,017 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:14:52,126 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:14:54,026 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:14:57,135 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:14:58,383 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 18:14:58,385 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 18:14:58,386 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 18:14:58,387 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 18:14:58,823 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 18:14:58,823 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 18:14:58,921 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 18:14:59,922 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 18:15:02,991 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:15:04,028 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:15:04,028 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:15:07,992 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:15:09,029 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:15:12,992 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:15:13,822 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 18:15:13,823 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 18:15:16,127 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 18:15:18,130 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:15:19,128 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:15:19,128 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:15:23,131 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:15:24,138 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:15:28,139 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:15:28,760 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 18:15:28,762 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 18:15:28,762 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 18:15:28,764 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 18:15:28,948 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 18:15:29,164 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:15:29,165 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:15:29,166 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 18:15:29,167 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 18:15:29,949 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 18:15:33,296 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:15:35,166 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:15:35,166 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:15:38,297 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:15:40,167 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:15:40,167 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:15:43,297 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:15:43,910 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 18:15:43,910 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 18:15:46,131 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 18:15:49,133 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:15:49,169 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:15:49,169 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:15:54,139 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:15:54,176 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:15:57,861 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 18:15:57,863 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 18:15:57,864 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 18:15:57,865 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 18:15:57,972 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 18:15:58,910 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 18:15:58,911 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 18:15:59,973 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 18:16:00,080 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:16:04,254 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:16:05,080 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:16:09,255 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:16:10,081 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:16:13,910 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 18:16:13,911 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 18:16:16,077 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:16:16,134 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 18:16:19,304 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:16:21,137 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:16:24,318 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:16:26,147 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:16:27,778 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 18:16:27,780 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 18:16:27,780 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 18:16:27,781 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 18:16:28,005 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 18:16:28,910 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 18:16:28,911 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 18:16:30,007 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 18:16:32,074 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:16:34,312 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:16:37,074 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:16:39,313 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:16:42,075 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:16:43,910 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 18:16:43,911 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 18:16:46,139 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 18:16:47,141 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:16:49,890 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:16:49,890 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:16:52,141 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:16:54,895 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:16:57,145 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:16:58,910 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 18:16:58,911 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 18:16:59,488 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 18:16:59,490 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 18:16:59,490 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 18:16:59,491 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 18:17:00,035 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 18:17:02,036 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 18:17:02,492 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:17:04,901 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:17:07,493 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:17:09,901 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:17:12,494 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:17:13,911 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 18:17:13,911 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 18:17:16,142 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 18:17:18,145 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:17:19,903 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:17:23,145 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:17:24,903 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:17:28,151 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:17:28,914 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 18:17:28,915 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 18:17:32,899 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 18:17:32,901 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 18:17:32,901 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 18:17:32,903 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 18:17:33,070 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 18:17:33,904 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:17:34,043 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:17:34,044 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:17:34,070 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 18:17:38,905 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:17:39,044 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:17:43,905 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:17:43,911 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 18:17:43,911 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 18:17:46,146 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 18:17:49,136 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:17:49,137 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:17:49,149 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:17:54,137 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:17:54,149 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:17:58,914 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 18:17:58,914 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 18:17:59,998 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:18:03,622 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 18:18:03,624 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 18:18:03,624 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 18:18:03,625 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 18:18:04,100 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 18:18:04,142 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:18:05,627 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:18:06,102 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 18:18:09,143 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:18:10,628 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:18:13,914 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 18:18:13,915 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 18:18:16,080 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:18:16,151 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 18:18:19,144 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:18:21,154 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:18:24,145 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:18:26,155 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:18:28,914 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 18:18:28,914 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 18:18:32,060 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:18:33,629 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 18:18:33,631 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 18:18:33,631 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 18:18:33,633 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 18:18:34,129 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 18:18:34,154 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:18:36,130 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 18:18:37,633 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:18:39,155 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:18:42,634 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:18:43,914 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 18:18:43,915 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 18:18:46,154 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 18:18:48,157 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:18:49,156 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:18:53,158 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:18:54,157 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:18:58,161 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:18:58,917 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 18:18:58,917 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 18:19:03,545 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 18:19:03,547 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 18:19:03,547 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 18:19:03,547 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:19:03,548 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 18:19:04,161 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 18:19:04,162 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:19:06,163 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 18:19:08,549 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:19:09,163 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:19:13,550 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:19:13,917 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 18:19:13,918 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 18:19:16,158 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 18:19:19,161 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:19:19,164 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:19:24,161 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:19:24,165 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:19:28,917 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 18:19:28,918 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 18:19:30,038 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:19:34,172 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:19:34,742 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 18:19:34,744 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 18:19:34,745 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 18:19:34,746 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 18:19:35,194 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 18:19:35,747 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:19:36,194 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 18:19:39,173 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:19:40,748 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:19:43,918 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 18:19:43,918 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 18:19:46,002 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:19:46,162 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 18:19:49,174 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:19:51,165 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:19:54,175 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:19:56,165 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:19:58,917 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 18:19:58,918 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 18:20:01,992 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:20:04,191 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:20:06,997 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:20:07,848 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 18:20:07,849 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 18:20:07,850 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 18:20:07,852 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 18:20:08,216 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 18:20:09,260 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:20:10,217 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 18:20:12,853 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:20:13,918 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 18:20:13,918 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 18:20:16,165 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 18:20:18,167 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:20:19,262 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:20:19,262 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:20:23,167 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:20:24,263 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:20:24,263 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:20:28,168 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:20:28,918 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 18:20:28,918 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 18:20:34,080 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:20:34,272 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:20:38,884 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 18:20:38,886 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 18:20:38,887 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 18:20:38,888 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 18:20:39,246 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 18:20:39,283 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:20:39,284 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:20:39,888 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:20:40,247 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 18:20:43,918 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 18:20:43,918 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 18:20:44,996 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:20:46,170 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 18:20:49,285 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:20:49,285 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:20:50,172 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:20:54,286 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:20:54,287 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:20:55,173 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:20:58,918 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 18:20:58,918 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 18:21:01,025 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:21:04,295 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:21:06,031 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:21:09,291 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:21:09,859 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 18:21:09,861 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 18:21:09,861 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 18:21:09,862 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 18:21:10,270 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 18:21:11,863 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:21:12,271 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 18:21:13,918 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 18:21:13,919 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 18:21:16,173 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 18:21:17,175 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:21:19,293 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:21:22,176 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:21:24,294 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:21:27,176 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:21:28,918 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 18:21:28,919 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 18:21:32,995 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:21:34,295 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:21:38,002 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:21:39,302 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:21:40,840 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 18:21:40,842 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 18:21:40,842 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 18:21:40,844 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 18:21:41,295 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 18:21:42,296 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 18:21:43,845 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:21:43,918 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 18:21:43,919 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 18:21:46,177 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 18:21:48,997 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:21:49,179 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:21:53,998 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:21:54,180 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:21:58,918 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 18:21:58,919 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 18:22:00,066 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:22:03,999 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:22:05,066 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:22:09,006 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:22:09,007 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:22:10,075 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:22:12,495 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 18:22:12,496 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 18:22:12,497 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 18:22:12,499 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 18:22:13,321 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 18:22:13,919 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 18:22:13,919 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 18:22:14,322 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 18:22:16,056 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:22:16,181 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 18:22:19,007 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:22:21,184 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:22:24,008 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:22:24,009 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:22:26,184 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:22:28,919 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 18:22:28,919 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 18:22:32,040 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:22:34,020 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:22:37,045 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:22:38,222 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 18:22:38,224 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 18:22:38,224 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 18:22:38,226 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 18:22:38,346 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 18:22:39,015 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:22:40,348 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 18:22:42,227 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:22:43,919 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 18:22:43,919 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 18:22:46,185 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 18:22:48,187 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:22:49,019 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:22:49,020 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:22:53,188 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:22:54,062 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:22:58,188 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:22:58,919 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 18:22:58,919 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 18:23:04,059 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:23:04,072 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:23:09,062 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:23:09,069 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:23:09,069 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:23:09,214 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 18:23:09,216 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 18:23:09,216 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 18:23:09,217 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 18:23:09,371 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 18:23:10,372 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 18:23:13,919 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 18:23:13,920 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 18:23:14,065 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:23:16,189 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 18:23:19,071 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:23:19,071 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:23:19,191 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:23:24,072 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:23:24,072 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:23:24,192 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:23:28,919 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 18:23:28,920 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 18:23:30,088 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:23:34,077 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:23:35,092 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:23:38,886 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 18:23:38,888 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 18:23:38,888 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 18:23:38,889 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 18:23:39,289 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:23:39,399 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 18:23:40,400 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 18:23:40,890 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:23:43,919 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 18:23:43,920 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 18:23:46,022 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:23:46,193 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 18:23:49,291 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:23:49,291 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:23:51,195 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:23:54,292 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:23:54,292 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:23:56,196 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:23:58,919 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 18:23:58,920 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 18:24:02,000 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:24:04,301 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:24:07,002 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:24:07,126 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 18:24:07,128 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 18:24:07,129 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 18:24:07,130 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 18:24:07,435 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 18:24:08,436 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 18:24:09,528 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:24:12,132 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:24:13,920 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 18:24:13,920 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 18:24:16,197 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 18:24:17,199 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:24:19,529 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:24:22,200 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:24:24,530 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:24:27,200 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:24:28,920 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 18:24:28,920 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 18:24:33,012 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:24:34,538 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:24:38,016 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:24:38,579 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 18:24:38,580 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 18:24:38,581 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 18:24:38,582 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 18:24:39,459 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 18:24:39,538 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:24:40,460 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 18:24:43,583 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:24:43,919 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 18:24:43,920 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 18:24:46,201 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 18:24:49,204 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:24:49,539 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:24:54,204 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:24:54,540 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:24:58,920 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 18:24:58,921 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 18:25:00,046 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:25:04,543 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:25:05,050 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:25:09,342 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 18:25:09,344 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 18:25:09,345 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 18:25:09,346 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 18:25:09,483 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 18:25:09,744 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:25:10,347 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:25:10,484 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 18:25:13,920 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 18:25:13,921 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 18:25:16,039 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:25:16,205 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 18:25:19,750 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:25:19,750 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:25:21,208 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:25:24,757 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:25:24,758 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:25:26,208 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:25:28,920 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 18:25:28,921 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 18:25:32,026 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:25:34,759 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:25:34,759 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:25:37,034 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:25:39,766 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:25:41,290 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 18:25:41,292 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 18:25:41,292 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 18:25:41,294 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 18:25:41,513 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 18:25:42,295 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:25:42,514 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 18:25:43,920 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 18:25:43,921 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 18:25:46,209 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 18:25:48,212 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:25:49,294 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:25:49,295 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:25:53,212 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:25:54,295 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:25:54,296 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:25:58,213 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:25:58,921 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 18:25:58,921 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 18:26:04,030 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:26:04,297 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:26:04,297 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:26:09,037 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:26:09,307 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:26:11,284 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 18:26:11,285 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 18:26:11,286 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 18:26:11,287 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 18:26:11,536 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 18:26:12,538 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 18:26:13,921 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 18:26:13,922 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 18:26:15,015 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:26:16,213 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 18:26:19,489 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:26:20,216 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:26:24,490 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:26:25,216 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:26:28,921 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 18:26:28,921 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 18:26:31,044 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:26:34,491 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:26:36,045 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:26:39,499 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:26:41,053 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:26:42,699 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 18:26:42,700 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 18:26:42,701 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 18:26:42,702 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 18:26:43,564 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 18:26:43,921 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 18:26:43,921 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 18:26:44,565 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 18:26:46,071 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:26:46,216 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 18:26:49,499 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:26:51,219 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:26:54,499 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:26:56,219 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:26:58,921 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 18:26:58,922 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 18:27:02,030 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:27:04,513 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:27:04,513 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:27:07,030 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:27:09,528 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:27:12,038 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:27:13,078 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 18:27:13,079 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 18:27:13,080 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 18:27:13,081 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 18:27:13,589 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 18:27:13,929 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 18:27:13,930 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 18:27:14,590 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 18:27:16,220 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 18:27:17,222 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:27:19,522 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:27:22,223 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:27:24,523 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:27:27,224 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:27:28,921 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 18:27:28,922 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 18:27:33,048 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:27:34,524 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:27:38,055 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:27:39,529 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:27:43,057 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:27:43,326 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 18:27:43,328 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 18:27:43,328 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 18:27:43,330 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 18:27:43,618 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 18:27:43,922 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 18:27:43,922 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 18:27:44,619 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 18:27:46,224 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 18:27:48,227 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:27:49,729 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:27:49,729 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:27:53,227 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:27:54,730 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:27:54,730 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:27:58,228 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:27:58,922 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 18:27:58,922 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 18:28:04,060 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:28:04,731 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:28:04,732 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:28:09,066 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:28:09,740 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:28:10,973 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 18:28:10,975 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 18:28:10,975 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 18:28:10,976 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 18:28:11,648 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 18:28:12,649 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 18:28:13,922 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 18:28:13,923 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 18:28:15,031 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:28:16,228 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 18:28:19,124 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:28:20,231 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:28:24,125 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:28:24,125 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:28:25,231 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:28:28,922 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 18:28:28,922 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 18:28:31,088 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:28:34,126 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:28:34,126 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:28:36,092 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:28:39,135 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:28:41,095 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:28:41,241 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 18:28:41,243 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 18:28:41,243 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 18:28:41,244 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 18:28:41,626 ERROR gpu :316822 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-12-25 18:28:41,681 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 18:28:42,681 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 18:28:43,922 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 18:28:43,923 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 18:28:46,106 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:28:46,233 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 18:28:49,526 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:28:49,527 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:28:51,236 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:28:54,527 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:28:54,527 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:28:56,236 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:28:58,922 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 18:28:58,923 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 18:29:02,031 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:29:04,529 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:29:04,529 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:29:07,039 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:29:09,530 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:29:12,038 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:29:12,360 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 18:29:12,361 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 18:29:12,362 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 18:29:12,363 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 18:29:12,711 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 18:29:13,922 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 18:29:13,923 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 18:29:14,712 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 18:29:16,236 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 18:29:17,238 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:29:19,035 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:29:19,035 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:29:22,239 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:29:24,163 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:29:24,253 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:29:27,240 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:29:28,922 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 18:29:28,923 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 18:29:33,081 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:29:34,254 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:29:38,086 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:29:39,264 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:29:41,653 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 18:29:41,655 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 18:29:41,656 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 18:29:41,657 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 18:29:41,738 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 18:29:42,739 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 18:29:43,659 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:29:43,923 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 18:29:43,923 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 18:29:46,240 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 18:29:49,243 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:29:49,686 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:29:49,686 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:29:54,243 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:29:54,687 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:29:54,687 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:29:58,923 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 18:29:58,923 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 18:30:00,079 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:30:04,689 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:30:04,689 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:30:05,080 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:30:09,695 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:30:10,088 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:30:12,026 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 18:30:12,027 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 18:30:12,028 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 18:30:12,030 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 18:30:12,774 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 18:30:12,775 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 18:30:13,923 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 18:30:13,924 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 18:30:16,077 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:30:16,244 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 18:30:19,879 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:30:21,246 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:30:24,880 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:30:26,247 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:30:28,923 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 18:30:28,924 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 18:30:31,997 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:30:34,881 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:30:34,881 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:30:36,998 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:30:39,891 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:30:42,001 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:30:43,923 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 18:30:43,924 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 18:30:44,036 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 18:30:44,038 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 18:30:44,038 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 18:30:44,040 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 18:30:44,803 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 18:30:44,803 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 18:30:46,247 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 18:30:47,249 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:30:49,891 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:30:52,249 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:30:54,892 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:30:57,250 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:30:58,923 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 18:30:58,924 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 18:31:03,008 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:31:03,933 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:31:08,009 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:31:08,934 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:31:08,934 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:31:13,013 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:31:13,931 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 18:31:13,932 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 18:31:16,251 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 18:31:16,392 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 18:31:16,394 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 18:31:16,394 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 18:31:16,395 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 18:31:16,834 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 18:31:18,396 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:31:18,836 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 18:31:18,942 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:31:18,943 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:31:23,397 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:31:23,943 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:31:28,397 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:31:28,927 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 18:31:28,928 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 18:31:34,000 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:31:34,168 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:31:34,168 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:31:39,001 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:31:39,168 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:31:39,169 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:31:43,931 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 18:31:43,931 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 18:31:44,284 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:31:46,256 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 18:31:46,647 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 18:31:46,649 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 18:31:46,650 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 18:31:46,651 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 18:31:46,861 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 18:31:48,863 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 18:31:49,176 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:31:49,176 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:31:49,652 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:31:54,177 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:31:54,177 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:31:54,653 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:31:58,931 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 18:31:58,931 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 18:32:00,064 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:32:04,178 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:32:04,178 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:32:05,065 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:32:09,179 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:32:09,179 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:32:10,065 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:32:13,931 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 18:32:13,934 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 18:32:16,010 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:32:16,261 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 18:32:17,410 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 18:32:17,411 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 18:32:17,412 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 18:32:17,413 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 18:32:17,887 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 18:32:18,888 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 18:32:19,186 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:32:21,415 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:32:24,187 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:32:24,187 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:32:26,416 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:32:29,099 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 18:32:29,100 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 18:32:32,248 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:32:34,189 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:32:34,189 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:32:37,249 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:32:39,189 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:32:39,190 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:32:42,306 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:32:44,011 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 18:32:44,017 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 18:32:46,263 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 18:32:46,540 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 18:32:46,542 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 18:32:46,542 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 18:32:46,544 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 18:32:46,910 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 18:32:47,545 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:32:48,911 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 18:32:49,201 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:32:52,546 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:32:54,202 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:32:57,546 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:32:59,010 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 18:32:59,010 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 18:33:03,184 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:33:04,203 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:33:08,185 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:33:09,204 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:33:13,192 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:33:14,013 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 18:33:14,014 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 18:33:16,264 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 18:33:16,469 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 18:33:16,471 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 18:33:16,471 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 18:33:16,473 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 18:33:16,935 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 18:33:18,474 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:33:18,936 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 18:33:19,208 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:33:23,474 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:33:24,208 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:33:28,475 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:33:29,012 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 18:33:29,012 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 18:33:34,270 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:33:34,270 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:33:34,332 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:33:39,270 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:33:39,271 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:33:39,333 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:33:44,018 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 18:33:44,020 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 18:33:45,113 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:33:46,267 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 18:33:46,833 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 18:33:46,834 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 18:33:46,834 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 18:33:46,835 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 18:33:46,967 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 18:33:48,970 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 18:33:49,283 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:33:49,283 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:33:50,836 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:33:54,283 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:33:55,837 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:33:59,015 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 18:33:59,016 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 18:34:01,093 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:34:04,285 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:34:04,285 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:34:06,093 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:34:09,291 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:34:11,101 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:34:12,978 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 18:34:12,979 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 18:34:12,980 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 18:34:12,981 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 18:34:12,996 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 18:34:14,021 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 18:34:14,022 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 18:34:14,998 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 18:34:16,107 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:34:16,272 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 18:34:19,382 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:34:19,382 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:34:21,274 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:34:24,383 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:34:26,275 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:34:29,021 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 18:34:29,022 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 18:34:32,129 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:34:34,384 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:34:37,131 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:34:39,398 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:34:42,142 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:34:42,632 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 18:34:42,634 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 18:34:42,634 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 18:34:42,635 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 18:34:43,023 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 18:34:44,021 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 18:34:44,022 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 18:34:45,029 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 18:34:46,275 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 18:34:47,277 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:34:49,391 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:34:52,278 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:34:54,391 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:34:57,279 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:34:59,021 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 18:34:59,022 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 18:35:03,179 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:35:04,438 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:35:08,182 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:35:09,444 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:35:09,445 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:35:13,186 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:35:13,337 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 18:35:13,338 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 18:35:13,339 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 18:35:13,340 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 18:35:14,021 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 18:35:14,022 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 18:35:14,055 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 18:35:16,057 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 18:35:16,279 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 18:35:18,282 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:35:19,807 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:35:19,808 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:35:23,282 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:35:24,808 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:35:24,808 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:35:28,283 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:35:29,022 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 18:35:29,022 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 18:35:34,160 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:35:34,810 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:35:34,810 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:35:39,165 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:35:39,816 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:35:43,400 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 18:35:43,402 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 18:35:43,402 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 18:35:43,403 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 18:35:44,022 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 18:35:44,023 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 18:35:44,083 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 18:35:45,161 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:35:46,084 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 18:35:46,283 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 18:35:49,819 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:35:49,819 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:35:50,286 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:35:54,820 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:35:54,820 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:35:55,286 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:35:59,022 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 18:35:59,023 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 18:36:01,164 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:36:04,821 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:36:04,822 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:36:06,164 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:36:09,837 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:36:11,169 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:36:14,022 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 18:36:14,022 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 18:36:15,051 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 18:36:15,052 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 18:36:15,052 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 18:36:15,054 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 18:36:15,109 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 18:36:16,287 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 18:36:16,294 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:36:18,111 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 18:36:19,832 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:36:21,290 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:36:24,833 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:36:26,291 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:36:29,022 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 18:36:29,023 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 18:36:32,154 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:36:34,834 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:36:37,155 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:36:39,844 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:36:42,161 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:36:44,028 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 18:36:44,029 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 18:36:45,078 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 18:36:45,079 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 18:36:45,079 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 18:36:45,081 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 18:36:45,142 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 18:36:46,143 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 18:36:46,292 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 18:36:47,294 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:36:49,840 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:36:52,295 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:36:54,841 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:36:57,296 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:36:59,029 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 18:36:59,029 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 18:37:03,151 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:37:04,843 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:37:08,151 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:37:09,849 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:37:13,159 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:37:14,034 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 18:37:14,034 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 18:37:15,381 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 18:37:15,382 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 18:37:15,383 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 18:37:15,385 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 18:37:16,176 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 18:37:16,295 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 18:37:18,179 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 18:37:18,298 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:37:19,852 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:37:23,298 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:37:24,853 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:37:28,299 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:37:29,034 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 18:37:29,034 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 18:37:34,164 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:37:34,854 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:37:39,164 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:37:39,855 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:37:44,039 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 18:37:44,048 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 18:37:44,214 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:37:46,296 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 18:37:47,164 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 18:37:47,165 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 18:37:47,165 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 18:37:47,170 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 18:37:47,209 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 18:37:48,210 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 18:37:49,859 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:37:50,171 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:37:54,860 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:37:55,172 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:37:59,037 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 18:37:59,038 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 18:38:00,184 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:38:04,862 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:38:05,185 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:38:09,863 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:38:10,186 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:38:14,046 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 18:38:14,047 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 18:38:16,133 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:38:16,299 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 18:38:16,304 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 18:38:16,306 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 18:38:16,307 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 18:38:16,308 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 18:38:17,233 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 18:38:18,234 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 18:38:19,872 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:38:21,309 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:38:24,872 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:38:26,310 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:38:29,039 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 18:38:29,039 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 18:38:32,204 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:38:34,874 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:38:37,205 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:38:39,875 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:38:42,211 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:38:44,054 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 18:38:44,058 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 18:38:46,302 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 18:38:47,303 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:38:47,355 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 18:38:47,356 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 18:38:47,356 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 18:38:47,357 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 18:38:48,260 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 18:38:48,260 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 18:38:49,884 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:38:49,884 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:38:52,358 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:38:54,884 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:38:57,359 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:38:59,046 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 18:38:59,046 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 18:39:03,185 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:39:04,886 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:39:08,186 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:39:09,887 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:39:13,198 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:39:14,046 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 18:39:14,054 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 18:39:16,307 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 18:39:16,796 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 18:39:16,797 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 18:39:16,797 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 18:39:16,799 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 18:39:17,287 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 18:39:18,288 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 18:39:18,800 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:39:19,197 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:39:19,198 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:39:23,800 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:39:24,198 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:39:24,198 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:39:28,801 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:39:29,047 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 18:39:29,047 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 18:39:34,138 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:39:34,200 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:39:39,138 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:39:39,200 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:39:39,201 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:39:44,053 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 18:39:44,055 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 18:39:44,177 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:39:46,312 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 18:39:46,999 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 18:39:47,000 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 18:39:47,001 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 18:39:47,003 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 18:39:47,314 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 18:39:48,314 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 18:39:49,404 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:39:49,404 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:39:50,004 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:39:54,405 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:39:54,405 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:39:55,004 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:39:59,050 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 18:39:59,051 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 18:40:00,124 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:40:04,407 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:40:04,407 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:40:05,125 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:40:09,408 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:40:09,408 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:40:10,126 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:40:14,056 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 18:40:14,057 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 18:40:15,214 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:40:16,318 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 18:40:18,027 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 18:40:18,028 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 18:40:18,028 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 18:40:18,030 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 18:40:18,338 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 18:40:19,427 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:40:20,340 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 18:40:21,031 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:40:24,428 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:40:24,428 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:40:26,032 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:40:29,054 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 18:40:29,055 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 18:40:31,133 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:40:34,430 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:40:34,430 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:40:36,133 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:40:39,431 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:40:39,431 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:40:41,134 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:40:44,065 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 18:40:44,067 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 18:40:46,153 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:40:46,329 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 18:40:49,438 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:40:49,556 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 18:40:49,558 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 18:40:49,558 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 18:40:49,560 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 18:40:50,371 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 18:40:51,561 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:40:52,373 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 18:40:54,439 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:40:56,562 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:40:59,056 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 18:40:59,057 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 18:41:02,184 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:41:04,440 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:41:07,185 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:41:09,441 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:41:12,186 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:41:14,056 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 18:41:14,057 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 18:41:16,326 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 18:41:17,329 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:41:19,449 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:41:22,336 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:41:23,187 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 18:41:23,188 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 18:41:23,189 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 18:41:23,194 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 18:41:23,402 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 18:41:24,403 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 18:41:24,582 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:41:28,195 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:41:29,057 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 18:41:29,057 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 18:41:33,218 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:41:34,583 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:41:38,218 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:41:39,596 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:41:39,596 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:41:43,219 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:41:44,056 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 18:41:44,057 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 18:41:46,330 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 18:41:48,337 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:41:49,602 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:41:52,951 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 18:41:52,952 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 18:41:52,953 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 18:41:52,955 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 18:41:53,434 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 18:41:53,956 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:41:54,435 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 18:41:54,607 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:41:58,957 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:41:59,057 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 18:41:59,057 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 18:42:04,179 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:42:04,608 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:42:09,179 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:42:09,609 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:42:14,057 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 18:42:14,057 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 18:42:15,166 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:42:16,332 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 18:42:19,626 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:42:20,340 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:42:22,545 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 18:42:22,548 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 18:42:22,549 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 18:42:22,550 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 18:42:23,472 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 18:42:24,473 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 18:42:24,693 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:42:24,693 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:42:25,551 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:42:29,057 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 18:42:29,057 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 18:42:31,118 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:42:34,694 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:42:36,119 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:42:39,695 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:42:41,120 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:42:44,057 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 18:42:44,058 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 18:42:46,192 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:42:46,336 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 18:42:49,702 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:42:51,345 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:42:53,901 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 18:42:53,903 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 18:42:53,904 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 18:42:53,909 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 18:42:54,509 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 18:42:54,701 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:42:56,511 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 18:42:56,910 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:42:59,057 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 18:42:59,058 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 18:43:02,155 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:43:04,702 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:43:07,155 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:43:09,703 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:43:12,156 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:43:14,057 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 18:43:14,058 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 18:43:16,340 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 18:43:17,342 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:43:19,704 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:43:22,343 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:43:24,717 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:43:27,348 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:43:29,063 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 18:43:29,063 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 18:43:30,369 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 18:43:30,370 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 18:43:30,370 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 18:43:30,376 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 18:43:30,540 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 18:43:32,377 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:43:32,541 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 18:43:34,754 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:43:34,755 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:43:37,377 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:43:39,755 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:43:39,756 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:43:42,378 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:43:44,278 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 18:43:44,279 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 18:43:46,344 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 18:43:48,346 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:43:49,757 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:43:49,757 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:43:53,347 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:43:54,758 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:43:54,758 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:43:58,352 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:43:59,195 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 18:43:59,196 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 18:44:02,140 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 18:44:02,142 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 18:44:02,142 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 18:44:02,146 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 18:44:02,575 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 18:44:04,147 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:44:04,576 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 18:44:04,766 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:44:09,147 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:44:09,767 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:44:14,148 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:44:14,188 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 18:44:14,189 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 18:44:16,348 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 18:44:19,351 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:44:19,768 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:44:24,351 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:44:24,769 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:44:29,193 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 18:44:29,196 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 18:44:30,347 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:44:34,253 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 18:44:34,255 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 18:44:34,255 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 18:44:34,257 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 18:44:34,611 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 18:44:34,778 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:44:36,258 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:44:36,613 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 18:44:39,779 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:44:41,259 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:44:44,193 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 18:44:44,194 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 18:44:46,266 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:44:46,352 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 18:44:49,894 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:44:49,894 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:44:51,354 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:44:54,895 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:44:54,895 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:44:56,355 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:44:59,205 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 18:44:59,210 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 18:45:01,373 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:45:04,811 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 18:45:04,812 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 18:45:04,813 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 18:45:04,817 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 18:45:04,956 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:45:05,205 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:45:05,637 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 18:45:06,638 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 18:45:06,817 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:45:10,205 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:45:10,206 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:45:11,818 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:45:14,195 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 18:45:14,196 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 18:45:16,357 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 18:45:17,359 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:45:19,207 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:45:19,207 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:45:22,359 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:45:24,208 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:45:24,208 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:45:27,360 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:45:29,195 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 18:45:29,196 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 18:45:33,278 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:45:34,218 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:45:36,619 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 18:45:36,621 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 18:45:36,621 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 18:45:36,626 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 18:45:36,665 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 18:45:38,627 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:45:38,667 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 18:45:39,218 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:45:43,627 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:45:44,195 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 18:45:44,196 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 18:45:46,360 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 18:45:49,219 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:45:49,362 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:45:54,220 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:45:54,363 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:45:59,196 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 18:45:59,196 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 18:46:00,332 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:46:04,221 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:46:05,332 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:46:09,226 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:46:10,339 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:46:12,339 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 18:46:12,341 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 18:46:12,341 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 18:46:12,346 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 18:46:12,694 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 18:46:14,196 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 18:46:14,196 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 18:46:14,696 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 18:46:15,354 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:46:16,364 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 18:46:19,233 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:46:20,366 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:46:24,234 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:46:25,367 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:46:29,196 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 18:46:29,197 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 18:46:31,279 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:46:34,235 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:46:36,279 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:46:39,236 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:46:41,283 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:46:44,208 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 18:46:44,212 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 18:46:46,229 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 18:46:46,230 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 18:46:46,230 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 18:46:46,235 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 18:46:46,368 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 18:46:46,370 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:46:46,729 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 18:46:48,731 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 18:46:49,375 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:46:49,375 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:46:51,371 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:46:54,376 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:46:56,371 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:46:59,201 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 18:46:59,202 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 18:47:02,295 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:47:04,377 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:47:07,295 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:47:09,378 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:47:12,303 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:47:14,208 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 18:47:14,210 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 18:47:16,373 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 18:47:16,554 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 18:47:16,555 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 18:47:16,556 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 18:47:16,557 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 18:47:16,755 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 18:47:17,558 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:47:18,756 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 18:47:19,382 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:47:22,559 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:47:24,383 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:47:27,559 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:47:29,202 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 18:47:29,203 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 18:47:33,332 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:47:34,384 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:47:38,332 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:47:39,385 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:47:43,345 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:47:44,211 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 18:47:44,211 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 18:47:46,384 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 18:47:48,386 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:47:48,881 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 18:47:48,883 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 18:47:48,883 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 18:47:48,886 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 18:47:49,389 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:47:49,787 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 18:47:50,787 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 18:47:53,888 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:47:54,390 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:47:58,888 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:47:59,206 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 18:47:59,206 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 18:48:04,371 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:48:04,468 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:48:09,372 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:48:09,469 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:48:14,209 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 18:48:14,210 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 18:48:15,338 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:48:16,384 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 18:48:17,928 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 18:48:17,929 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 18:48:17,929 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 18:48:17,931 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 18:48:18,811 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 18:48:18,811 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 18:48:19,476 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:48:20,932 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:48:24,477 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:48:25,932 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:48:29,206 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 18:48:29,207 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 18:48:31,324 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:48:34,582 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:48:34,583 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:48:36,324 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:48:39,583 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:48:39,583 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:48:41,328 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:48:44,218 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 18:48:44,219 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 18:48:46,387 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 18:48:46,388 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:48:47,009 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 18:48:47,010 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 18:48:47,010 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 18:48:47,011 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 18:48:47,837 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 18:48:48,838 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 18:48:49,591 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:48:52,012 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:48:54,592 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:48:57,013 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:48:59,212 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 18:48:59,213 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 18:49:02,342 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:49:04,597 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:49:07,342 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:49:09,600 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:49:12,349 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:49:14,214 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 18:49:14,214 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 18:49:15,388 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 18:49:15,390 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 18:49:15,390 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 18:49:15,392 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 18:49:15,869 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 18:49:16,390 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 18:49:16,870 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 18:49:17,392 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:49:19,792 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:49:19,792 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:49:22,392 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:49:24,792 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:49:27,393 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:49:29,214 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 18:49:29,215 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 18:49:33,292 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:49:34,872 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:49:34,873 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:49:38,293 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:49:39,886 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:49:43,300 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:49:44,215 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 18:49:44,216 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 18:49:44,392 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 18:49:44,394 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 18:49:44,394 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 18:49:44,396 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 18:49:44,894 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 18:49:46,393 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 18:49:46,896 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 18:49:48,396 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:49:49,884 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:49:53,396 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:49:54,885 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:49:58,397 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:49:59,214 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 18:49:59,215 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 18:50:04,379 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:50:04,887 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:50:09,380 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:50:09,894 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:50:14,223 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 18:50:14,224 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 18:50:15,372 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:50:15,412 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 18:50:15,414 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 18:50:15,415 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 18:50:15,416 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 18:50:15,920 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 18:50:16,398 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 18:50:16,921 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 18:50:19,898 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:50:19,898 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:50:20,401 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:50:24,898 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:50:24,899 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:50:25,401 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:50:29,220 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 18:50:29,221 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 18:50:31,341 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:50:35,025 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:50:35,025 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:50:36,341 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:50:40,039 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:50:41,347 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:50:44,221 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 18:50:44,221 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 18:50:44,660 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 18:50:44,661 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 18:50:44,661 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 18:50:44,662 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 18:50:44,943 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 18:50:46,401 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 18:50:46,408 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:50:46,945 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 18:50:50,065 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:50:50,066 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:50:51,404 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:50:55,067 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:50:55,067 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:50:56,405 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:50:59,221 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 18:50:59,222 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 18:51:02,308 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:51:05,068 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:51:05,069 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:51:07,309 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:51:10,074 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:51:12,312 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:51:14,236 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 18:51:14,237 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 18:51:15,879 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 18:51:15,880 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 18:51:15,880 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 18:51:15,881 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 18:51:15,978 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 18:51:16,406 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 18:51:16,978 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 18:51:17,408 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:51:19,280 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:51:22,408 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:51:24,280 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:51:27,409 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:51:29,223 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 18:51:29,224 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 18:51:33,327 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:51:34,282 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:51:38,327 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:51:39,282 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:51:43,333 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:51:44,234 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 18:51:44,236 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 18:51:46,411 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 18:51:48,412 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:51:48,578 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 18:51:48,580 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 18:51:48,580 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 18:51:48,582 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 18:51:49,008 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 18:51:49,291 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:51:51,010 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 18:51:53,583 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:51:54,292 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:51:58,584 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:51:59,227 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 18:51:59,228 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 18:52:04,307 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:52:04,386 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:52:09,308 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:52:09,308 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:52:09,387 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:52:14,227 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 18:52:14,228 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 18:52:15,352 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:52:16,417 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 18:52:19,313 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:52:20,256 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 18:52:20,258 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 18:52:20,258 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 18:52:20,259 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 18:52:21,044 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 18:52:21,045 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 18:52:21,261 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:52:24,405 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:52:24,405 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:52:26,261 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:52:29,227 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 18:52:29,228 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 18:52:31,302 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:52:34,407 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:52:34,407 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:52:36,303 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:52:39,408 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:52:39,408 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:52:41,304 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:52:44,227 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 18:52:44,228 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 18:52:46,350 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:52:46,418 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 18:52:49,421 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:52:49,421 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:52:51,429 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:52:53,674 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 18:52:53,675 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 18:52:53,676 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 18:52:53,681 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 18:52:54,073 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 18:52:54,415 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:52:55,074 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 18:52:56,682 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:52:59,228 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 18:52:59,228 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 18:53:02,336 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:53:04,416 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:53:07,336 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:53:09,460 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:53:09,461 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:53:12,337 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:53:14,228 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 18:53:14,228 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 18:53:16,423 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 18:53:17,425 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:53:19,467 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:53:22,437 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:53:23,946 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 18:53:23,947 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 18:53:23,948 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 18:53:23,954 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 18:53:24,098 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 18:53:24,471 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:53:25,099 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 18:53:27,955 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:53:29,228 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 18:53:29,228 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 18:53:33,332 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:53:34,473 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:53:38,333 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:53:39,474 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:53:43,334 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:53:44,228 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 18:53:44,228 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 18:53:46,426 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 18:53:48,428 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:53:49,479 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:53:53,438 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:53:54,483 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:53:55,064 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 18:53:55,066 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 18:53:55,066 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 18:53:55,069 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 18:53:55,130 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 18:53:57,132 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 18:53:59,070 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:53:59,228 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 18:53:59,228 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 18:54:04,373 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:54:04,492 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:54:09,374 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:54:09,494 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:54:14,228 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 18:54:14,229 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 18:54:14,390 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:54:16,430 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 18:54:19,432 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:54:19,500 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:54:19,500 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:54:24,437 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:54:24,502 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:54:24,502 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:54:24,659 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 18:54:24,661 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 18:54:24,661 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 18:54:24,662 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 18:54:25,160 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 18:54:25,160 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 18:54:29,228 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 18:54:29,229 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 18:54:30,337 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:54:34,504 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:54:34,504 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:54:35,338 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:54:39,505 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:54:39,506 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:54:40,338 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:54:44,228 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 18:54:44,229 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 18:54:46,296 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:54:46,434 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 18:54:49,507 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:54:49,507 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:54:51,441 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:54:54,515 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:54:56,223 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 18:54:56,225 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 18:54:56,226 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 18:54:56,227 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 18:54:57,193 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 18:54:57,193 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 18:54:57,228 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:54:59,229 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 18:54:59,229 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 18:55:02,372 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:55:04,629 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:55:07,373 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:55:09,630 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:55:12,374 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:55:14,229 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 18:55:14,229 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 18:55:16,438 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 18:55:17,440 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:55:19,631 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:55:22,443 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:55:24,643 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:55:27,444 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:55:28,077 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 18:55:28,078 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 18:55:28,079 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 18:55:28,081 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 18:55:28,221 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 18:55:29,222 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 18:55:29,229 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 18:55:29,230 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 18:55:33,347 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:55:34,640 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:55:38,348 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:55:39,641 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:55:43,348 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:55:44,229 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 18:55:44,230 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 18:55:46,442 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 18:55:48,445 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:55:49,642 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:55:53,452 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:55:54,650 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:55:57,583 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 18:55:57,585 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 18:55:57,585 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 18:55:57,587 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 18:55:58,255 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 18:55:58,588 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:55:59,229 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 18:55:59,230 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 18:55:59,256 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 18:56:04,362 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:56:04,978 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:56:04,978 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:56:09,362 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:56:09,979 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:56:09,979 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:56:14,229 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 18:56:14,230 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 18:56:15,348 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:56:16,446 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 18:56:19,980 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:56:19,981 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:56:20,448 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:56:24,984 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:56:25,458 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:56:29,231 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 18:56:29,232 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 18:56:29,840 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 18:56:29,842 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 18:56:29,843 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 18:56:29,844 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 18:56:30,286 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 18:56:30,845 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:56:31,288 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 18:56:34,986 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:56:35,846 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:56:39,987 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:56:40,847 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:56:44,229 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 18:56:44,230 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 18:56:46,308 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:56:46,450 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 18:56:49,988 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:56:51,453 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:56:54,989 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:56:56,453 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:56:59,230 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 18:56:59,231 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 18:57:01,309 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 18:57:02,421 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:57:07,422 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:57:09,555 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:57:09,852 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:57:12,423 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:57:16,454 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 18:57:17,456 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:57:18,952 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:57:19,002 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:57:19,960 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 18:57:19,961 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 18:57:23,053 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:57:28,054 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:57:28,535 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:57:33,054 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:57:38,055 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:57:40,047 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:57:41,935 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:57:42,487 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 18:57:42,488 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 18:57:43,574 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:57:46,458 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 18:57:49,461 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:57:50,761 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:57:50,933 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:57:51,112 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 18:57:51,113 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 18:57:55,210 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:58:00,210 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:58:00,636 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:58:00,900 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:58:05,211 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:58:06,271 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 18:58:06,272 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 18:58:07,503 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 18:58:09,504 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 18:58:10,538 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:58:12,272 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:58:12,272 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:58:15,538 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:58:16,462 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 18:58:17,273 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:58:17,273 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:58:21,279 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 18:58:21,279 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 18:58:21,362 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:58:26,363 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:58:27,278 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:58:27,278 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:58:31,364 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:58:32,279 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:58:32,279 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:58:36,278 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 18:58:36,278 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 18:58:36,405 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:58:41,281 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:58:41,282 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:58:41,405 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:58:46,282 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:58:46,283 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:58:46,406 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:58:46,467 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 18:58:51,278 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 18:58:51,279 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 18:58:51,408 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:58:56,284 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:58:56,285 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:58:56,409 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:59:01,285 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:59:01,286 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:59:01,410 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:59:06,278 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 18:59:06,279 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 18:59:06,410 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:59:11,411 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:59:16,411 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:59:16,470 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 18:59:21,473 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:59:26,479 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:59:29,551 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:59:29,554 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:59:29,555 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 18:59:29,555 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 18:59:31,702 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:59:34,555 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:59:34,556 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:59:36,703 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:59:39,556 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:59:39,557 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:59:41,703 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:59:44,552 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 18:59:44,552 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 18:59:46,474 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 18:59:47,477 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:59:49,558 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:59:49,559 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:59:52,477 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:59:54,559 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:59:54,559 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 18:59:57,478 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 18:59:59,552 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 18:59:59,552 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 19:00:02,734 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:00:04,561 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:00:04,561 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:00:07,735 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:00:09,561 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:00:09,562 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:00:12,736 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:00:14,552 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 19:00:14,552 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 19:00:16,478 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 19:00:18,480 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:00:19,564 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:00:19,564 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:00:23,481 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:00:24,564 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:00:28,482 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:00:29,552 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 19:00:29,553 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 19:00:33,671 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:00:34,571 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:00:38,672 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:00:39,572 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:00:43,673 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:00:44,552 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 19:00:44,553 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 19:00:46,482 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 19:00:49,485 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:00:49,573 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:00:54,486 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:00:54,574 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:00:59,486 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:00:59,552 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 19:00:59,552 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 19:01:04,576 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:01:04,688 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:01:09,577 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:01:09,689 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:01:14,552 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 19:01:14,553 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 19:01:15,689 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:01:16,486 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 19:01:19,578 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:01:21,489 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:01:24,580 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:01:26,490 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:01:29,552 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 19:01:29,553 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 19:01:31,640 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:01:36,641 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:01:41,641 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:01:46,492 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 19:01:47,494 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:01:52,495 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:01:57,495 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:02:02,496 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:02:07,497 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:02:08,071 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:02:08,076 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 19:02:08,076 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 19:02:12,527 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 19:02:13,060 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:02:13,218 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:02:13,218 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:02:16,493 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 19:02:16,529 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 19:02:18,219 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:02:18,219 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:02:18,495 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:02:23,113 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 19:02:23,114 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 19:02:24,253 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:02:28,221 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:02:28,221 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:02:29,253 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:02:33,221 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:02:33,222 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:02:34,254 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:02:38,126 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 19:02:38,131 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 19:02:40,216 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:02:41,670 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 19:02:41,671 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 19:02:41,672 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 19:02:41,673 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 19:02:42,552 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 19:02:43,643 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:02:44,554 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 19:02:45,675 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:02:46,497 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 19:02:48,645 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:02:48,645 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:02:51,500 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:02:53,118 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 19:02:53,119 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 19:02:57,206 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:02:58,647 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:02:58,647 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:03:02,207 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:03:03,648 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:03:03,648 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:03:07,207 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:03:08,118 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 19:03:08,119 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 19:03:12,291 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:03:13,656 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:03:16,491 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 19:03:16,492 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 19:03:16,493 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 19:03:16,494 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 19:03:16,499 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 19:03:16,582 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 19:03:17,501 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:03:18,583 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 19:03:18,885 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:03:18,885 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:03:22,501 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:03:23,118 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 19:03:23,119 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 19:03:28,286 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:03:28,887 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:03:28,887 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:03:33,287 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:03:33,888 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:03:33,888 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:03:38,118 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 19:03:38,119 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 19:03:39,233 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:03:43,889 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:03:43,890 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:03:44,241 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:03:46,505 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 19:03:48,891 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:03:49,508 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:03:50,270 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 19:03:50,271 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 19:03:50,272 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 19:03:50,273 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 19:03:50,608 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 19:03:52,610 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 19:03:53,119 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 19:03:53,119 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 19:03:55,189 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:03:58,893 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:04:00,190 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:04:03,894 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:04:05,190 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:04:08,119 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 19:04:08,119 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 19:04:10,235 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:04:13,901 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:04:15,243 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:04:16,510 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 19:04:18,628 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 19:04:18,630 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 19:04:18,630 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 19:04:18,632 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 19:04:18,636 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 19:04:19,022 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:04:19,022 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:04:20,632 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:04:20,638 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 19:04:23,119 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 19:04:23,119 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 19:04:26,197 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:04:29,023 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:04:31,198 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:04:34,161 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:04:34,161 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:04:36,198 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:04:38,119 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 19:04:38,119 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 19:04:41,236 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:04:43,170 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:04:46,237 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:04:46,436 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 19:04:46,438 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 19:04:46,438 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 19:04:46,440 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 19:04:46,511 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 19:04:46,659 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 19:04:48,167 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:04:48,660 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 19:04:51,514 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:04:53,119 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 19:04:53,122 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 19:04:57,237 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:04:58,169 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:05:02,238 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:05:03,169 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:05:07,238 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:05:08,119 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 19:05:08,120 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 19:05:12,244 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:05:13,176 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:05:16,515 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 19:05:16,744 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 19:05:16,746 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 19:05:16,746 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 19:05:16,748 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 19:05:17,699 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 19:05:17,748 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:05:18,177 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:05:18,699 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 19:05:22,749 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:05:23,119 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 19:05:23,120 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 19:05:28,178 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:05:28,291 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:05:33,179 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:05:33,291 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:05:38,119 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 19:05:38,120 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 19:05:39,205 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:05:43,188 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:05:44,216 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:05:46,516 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 19:05:47,046 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 19:05:47,047 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 19:05:47,047 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 19:05:47,049 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 19:05:47,721 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 19:05:48,191 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:05:48,722 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 19:05:50,050 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:05:53,119 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 19:05:53,120 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 19:05:55,256 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:05:58,192 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:06:00,257 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:06:03,193 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:06:05,257 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:06:08,120 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 19:06:08,120 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 19:06:10,280 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:06:13,209 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:06:15,281 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:06:16,522 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 19:06:18,207 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:06:19,016 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 19:06:19,018 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 19:06:19,018 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 19:06:19,020 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 19:06:19,757 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 19:06:20,758 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 19:06:21,021 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:06:23,120 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 19:06:23,120 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 19:06:26,224 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:06:28,415 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:06:31,224 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:06:33,416 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:06:36,225 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:06:38,120 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 19:06:38,121 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 19:06:41,244 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:06:43,428 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:06:46,247 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:06:46,537 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 19:06:48,425 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:06:48,570 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 19:06:48,571 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 19:06:48,572 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 19:06:48,574 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 19:06:48,784 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 19:06:50,785 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 19:06:51,575 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:06:53,120 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 19:06:53,121 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 19:06:57,269 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:06:58,427 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:07:02,270 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:07:03,428 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:07:07,271 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:07:08,120 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 19:07:08,121 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 19:07:13,200 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:07:13,430 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:07:16,535 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 19:07:18,440 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:07:18,537 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:07:19,275 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 19:07:19,277 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 19:07:19,278 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 19:07:19,281 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 19:07:19,816 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 19:07:20,817 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 19:07:23,120 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 19:07:23,121 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 19:07:24,188 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:07:28,674 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:07:28,675 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:07:29,189 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:07:33,675 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:07:33,676 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:07:34,189 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:07:38,120 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 19:07:38,121 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 19:07:39,217 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:07:43,677 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:07:43,678 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:07:44,223 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:07:46,536 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 19:07:48,689 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:07:49,541 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:07:49,979 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 19:07:49,980 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 19:07:49,981 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 19:07:49,984 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 19:07:50,844 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 19:07:52,845 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 19:07:53,121 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 19:07:53,121 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 19:07:55,280 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:07:58,687 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:08:00,280 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:08:03,688 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:08:05,281 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:08:08,120 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 19:08:08,121 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 19:08:11,211 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:08:13,689 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:08:16,224 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:08:16,538 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 19:08:18,696 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:08:19,971 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 19:08:19,972 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 19:08:19,972 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 19:08:19,974 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 19:08:20,869 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 19:08:21,975 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:08:22,870 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 19:08:23,121 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 19:08:23,121 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 19:08:27,236 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:08:28,695 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:08:32,236 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:08:33,696 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:08:37,237 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:08:38,121 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 19:08:38,121 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 19:08:42,285 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:08:43,697 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:08:46,542 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 19:08:47,543 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:08:48,704 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:08:51,112 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 19:08:51,113 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 19:08:51,114 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 19:08:51,115 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 19:08:51,903 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 19:08:52,904 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 19:08:53,116 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:08:53,121 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 19:08:53,121 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 19:08:58,280 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:08:58,702 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:09:03,280 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:09:03,703 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:09:08,121 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 19:09:08,122 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 19:09:09,229 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:09:13,705 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:09:14,229 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:09:16,547 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 19:09:18,712 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:09:19,556 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:09:23,121 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 19:09:23,122 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 19:09:23,222 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 19:09:23,246 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 19:09:23,246 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 19:09:23,251 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 19:09:23,930 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 19:09:24,931 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 19:09:25,253 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:09:28,712 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:09:30,254 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:09:33,713 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:09:35,254 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:09:38,121 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 19:09:38,122 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 19:09:41,238 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:09:43,715 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:09:46,238 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:09:46,551 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 19:09:48,715 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:09:51,558 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:09:53,129 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 19:09:53,130 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 19:09:55,108 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 19:09:55,110 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 19:09:55,110 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 19:09:55,116 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 19:09:55,954 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 19:09:56,955 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 19:09:57,117 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:09:58,724 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:10:02,117 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:10:03,725 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:10:07,118 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:10:08,125 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 19:10:08,126 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 19:10:12,240 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:10:13,777 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:10:16,554 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 19:10:17,557 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:10:18,778 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:10:18,779 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:10:22,565 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:10:23,134 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 19:10:23,135 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 19:10:26,172 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 19:10:26,173 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 19:10:26,173 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 19:10:26,176 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 19:10:26,981 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 19:10:28,176 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:10:28,783 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:10:28,983 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 19:10:33,177 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:10:33,784 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:10:38,131 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 19:10:38,132 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 19:10:38,300 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:10:43,300 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:10:43,785 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:10:46,558 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 19:10:48,561 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:10:48,786 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:10:53,138 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 19:10:53,141 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 19:10:54,304 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:10:57,408 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 19:10:57,410 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 19:10:57,410 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 19:10:57,416 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 19:10:58,009 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 19:10:58,811 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:10:58,811 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:10:59,010 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 19:10:59,417 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:11:03,812 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:11:04,417 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:11:08,135 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 19:11:08,135 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 19:11:10,300 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:11:13,814 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:11:13,814 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:11:15,300 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:11:16,562 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 19:11:18,814 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:11:18,815 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:11:20,565 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:11:23,135 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 19:11:23,136 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 19:11:26,289 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:11:28,820 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:11:31,289 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:11:31,901 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 19:11:31,902 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 19:11:31,903 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 19:11:31,907 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 19:11:32,043 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 19:11:33,044 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 19:11:33,820 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:11:36,909 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:11:38,135 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 19:11:38,136 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 19:11:42,286 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:11:43,843 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:11:43,844 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:11:46,565 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 19:11:47,568 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:11:48,845 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:11:52,568 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:11:53,135 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 19:11:53,136 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 19:11:58,277 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:11:58,850 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:12:00,080 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 19:12:00,081 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 19:12:00,082 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 19:12:00,083 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 19:12:01,069 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 19:12:03,070 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 19:12:03,851 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:12:04,085 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:12:08,136 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 19:12:08,136 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 19:12:09,293 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:12:13,960 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:12:13,961 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:12:14,294 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:12:16,570 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 19:12:18,961 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:12:18,962 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:12:19,573 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:12:23,141 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 19:12:23,147 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 19:12:25,318 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:12:28,969 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:12:29,122 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 19:12:29,124 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 19:12:29,124 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 19:12:29,125 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 19:12:30,096 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 19:12:31,097 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 19:12:31,126 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:12:33,970 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:12:36,126 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:12:38,137 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 19:12:38,138 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 19:12:41,294 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:12:43,196 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:12:43,197 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:12:46,294 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:12:46,573 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 19:12:48,197 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:12:51,575 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:12:53,137 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 19:12:53,138 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 19:12:57,328 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:12:58,210 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:13:00,654 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 19:13:00,656 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 19:13:00,657 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 19:13:00,658 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 19:13:01,126 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 19:13:01,127 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 19:13:02,659 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:13:03,204 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:13:07,660 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:13:08,137 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 19:13:08,138 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 19:13:13,205 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:13:13,306 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:13:16,577 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 19:13:18,268 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:13:18,269 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:13:18,579 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:13:23,137 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 19:13:23,138 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 19:13:24,240 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:13:28,278 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:13:29,244 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:13:33,278 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:13:33,278 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:13:34,249 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:13:34,484 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 19:13:34,486 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 19:13:34,487 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 19:13:34,489 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 19:13:35,158 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 19:13:35,159 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 19:13:38,138 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 19:13:38,138 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 19:13:39,253 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:13:43,279 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:13:43,280 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:13:44,254 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:13:46,581 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 19:13:48,280 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:13:48,281 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:13:49,583 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:13:53,138 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 19:13:53,138 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 19:13:55,212 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:13:58,426 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:13:58,426 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:14:00,213 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:14:03,434 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:14:05,221 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:14:07,442 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 19:14:07,443 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 19:14:07,443 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 19:14:07,444 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 19:14:08,138 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 19:14:08,138 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 19:14:08,189 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 19:14:09,189 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 19:14:10,300 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:14:13,435 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:14:15,300 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:14:16,585 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 19:14:18,436 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:14:20,587 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:14:23,138 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 19:14:23,139 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 19:14:26,220 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:14:28,437 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:14:31,221 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:14:33,444 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:14:36,230 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:14:37,126 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 19:14:37,128 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 19:14:37,128 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 19:14:37,130 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 19:14:37,216 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 19:14:38,138 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 19:14:38,139 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 19:14:39,219 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 19:14:41,277 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:14:43,515 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:14:43,516 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:14:46,277 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:14:46,589 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 19:14:48,516 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:14:48,516 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:14:51,592 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:14:53,138 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 19:14:53,139 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 19:14:57,301 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:14:58,518 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:14:58,518 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:15:02,308 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:15:03,527 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:15:07,312 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:15:07,555 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 19:15:07,557 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 19:15:07,558 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 19:15:07,560 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 19:15:08,143 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 19:15:08,144 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 19:15:08,249 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 19:15:09,250 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 19:15:13,217 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:15:13,531 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:15:16,594 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 19:15:18,531 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:15:18,596 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:15:23,142 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 19:15:23,143 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 19:15:24,300 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:15:28,533 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:15:28,533 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:15:29,300 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:15:33,541 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:15:34,306 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:15:38,142 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 19:15:38,143 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 19:15:38,815 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 19:15:38,817 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 19:15:38,817 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 19:15:38,819 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 19:15:39,278 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 19:15:39,279 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 19:15:39,820 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:15:43,543 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:15:44,821 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:15:46,595 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 19:15:48,544 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:15:50,597 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:15:53,143 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 19:15:53,143 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 19:15:56,242 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:15:58,545 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:16:01,243 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:16:03,558 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:16:06,251 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:16:06,750 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 19:16:06,752 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 19:16:06,752 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 19:16:06,753 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 19:16:07,307 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 19:16:07,307 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 19:16:08,143 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 19:16:08,143 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 19:16:11,253 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:16:13,562 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:16:16,253 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:16:16,599 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 19:16:18,563 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:16:21,602 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:16:23,143 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 19:16:23,144 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 19:16:27,300 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:16:28,564 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:16:32,300 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:16:33,568 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:16:37,312 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:16:38,143 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 19:16:38,143 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 19:16:38,856 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 19:16:38,857 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 19:16:38,858 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 19:16:38,860 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 19:16:39,336 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 19:16:39,337 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 19:16:42,861 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:16:43,577 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:16:46,603 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 19:16:48,578 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:16:48,605 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:16:53,143 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 19:16:53,144 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 19:16:54,246 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:16:58,580 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:16:59,246 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:17:03,581 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:17:04,254 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:17:08,143 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 19:17:08,144 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 19:17:08,168 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 19:17:08,312 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 19:17:08,312 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 19:17:08,314 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 19:17:08,365 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 19:17:09,315 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:17:09,366 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 19:17:13,585 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:17:14,316 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:17:16,607 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 19:17:18,586 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:17:19,610 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:17:23,143 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 19:17:23,144 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 19:17:25,295 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:17:28,660 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:17:30,296 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:17:33,660 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:17:35,300 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:17:38,152 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 19:17:38,153 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 19:17:40,750 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 19:17:40,751 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 19:17:40,752 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 19:17:40,752 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:17:40,754 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 19:17:41,401 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 19:17:41,402 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 19:17:43,930 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:17:43,931 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:17:45,755 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:17:46,611 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 19:17:48,931 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:17:48,932 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:17:51,614 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:17:53,149 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 19:17:53,149 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 19:17:57,238 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:17:58,933 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:17:58,933 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:18:02,239 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:18:03,934 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:18:03,934 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:18:07,241 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:18:08,152 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 19:18:08,152 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 19:18:12,245 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:18:12,798 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 19:18:12,799 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 19:18:12,799 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 19:18:12,800 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 19:18:13,433 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 19:18:13,433 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 19:18:13,944 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:18:13,945 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:18:16,616 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 19:18:17,618 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:18:18,945 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:18:18,945 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:18:22,619 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:18:23,152 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 19:18:23,153 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 19:18:28,260 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:18:28,946 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:18:33,260 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:18:33,947 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:18:33,948 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:18:38,160 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 19:18:38,165 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 19:18:38,305 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:18:43,310 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:18:43,951 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:18:44,396 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 19:18:44,397 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 19:18:44,398 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 19:18:44,399 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 19:18:44,462 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 19:18:45,463 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 19:18:46,620 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 19:18:48,622 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:18:48,952 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:18:53,155 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 19:18:53,156 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 19:18:54,254 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:18:58,953 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:18:59,255 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:19:03,954 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:19:04,255 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:19:08,155 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 19:19:08,156 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 19:19:09,307 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:19:13,961 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:19:14,311 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:19:15,406 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 19:19:15,407 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 19:19:15,407 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 19:19:15,409 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 19:19:15,492 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 19:19:16,624 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 19:19:17,493 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 19:19:18,960 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:19:19,626 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:19:23,155 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 19:19:23,156 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 19:19:25,242 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:19:28,961 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:19:30,243 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:19:33,962 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:19:35,244 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:19:38,167 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 19:19:38,174 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 19:19:40,271 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:19:43,975 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:19:44,060 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 19:19:44,061 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 19:19:44,062 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 19:19:44,063 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 19:19:44,521 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 19:19:45,522 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 19:19:46,064 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:19:46,628 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 19:19:48,976 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:19:51,630 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:19:53,157 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 19:19:53,158 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 19:19:57,261 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:19:58,981 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:19:58,982 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:20:02,262 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:20:03,982 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:20:03,983 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:20:07,262 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:20:08,157 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 19:20:08,158 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 19:20:12,317 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:20:13,994 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:20:13,994 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:20:16,631 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 19:20:17,372 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 19:20:17,373 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 19:20:17,373 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 19:20:17,374 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:20:17,375 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 19:20:17,560 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 19:20:18,993 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:20:19,562 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 19:20:22,376 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:20:23,157 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 19:20:23,158 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 19:20:28,323 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:20:29,011 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:20:29,011 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:20:33,323 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:20:34,012 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:20:38,165 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 19:20:38,175 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 19:20:39,242 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:20:44,022 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:20:44,252 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:20:44,383 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 19:20:44,385 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 19:20:44,385 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 19:20:44,387 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 19:20:44,591 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 19:20:45,592 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 19:20:46,636 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 19:20:49,023 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:20:49,638 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:20:53,161 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 19:20:53,161 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 19:20:55,337 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:20:59,024 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:21:00,337 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:21:04,025 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:21:05,338 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:21:08,161 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 19:21:08,162 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 19:21:11,333 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:21:14,033 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:21:15,259 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 19:21:15,261 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 19:21:15,261 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 19:21:15,263 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 19:21:15,616 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 19:21:15,616 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 19:21:16,639 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 19:21:16,645 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:21:19,033 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:21:19,034 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:21:21,642 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:21:23,161 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 19:21:23,162 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 19:21:27,248 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:21:29,035 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:21:29,035 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:21:32,249 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:21:34,036 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:21:34,036 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:21:37,249 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:21:38,161 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 19:21:38,162 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 19:21:42,344 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:21:44,047 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:21:44,048 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:21:45,877 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 19:21:45,879 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 19:21:45,879 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 19:21:45,881 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 19:21:46,643 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 19:21:46,650 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 19:21:47,645 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:21:47,651 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 19:21:49,270 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:21:52,646 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:21:53,161 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 19:21:53,162 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 19:21:58,271 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:21:58,329 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:22:03,272 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:22:03,330 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:22:08,162 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 19:22:08,162 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 19:22:09,289 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:22:13,278 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:22:14,292 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:22:16,163 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 19:22:16,164 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 19:22:16,164 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 19:22:16,166 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 19:22:16,647 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 19:22:16,676 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 19:22:17,677 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 19:22:18,309 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:22:18,309 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:22:19,650 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:22:23,162 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 19:22:23,163 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 19:22:25,271 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:22:28,311 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:22:28,311 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:22:30,272 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:22:33,312 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:22:33,313 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:22:35,273 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:22:38,162 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 19:22:38,162 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 19:22:41,265 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:22:43,316 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:22:43,316 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:22:46,266 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:22:46,651 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 19:22:47,340 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 19:22:47,341 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 19:22:47,341 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 19:22:47,343 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 19:22:47,704 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 19:22:47,704 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 19:22:48,326 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:22:48,326 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:22:51,343 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:22:53,162 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 19:22:53,163 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 19:22:57,241 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:22:58,327 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:22:58,328 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:23:02,242 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:23:03,350 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:23:07,243 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:23:08,162 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 19:23:08,163 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 19:23:12,301 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:23:13,351 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:23:16,654 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 19:23:17,659 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:23:18,357 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:23:22,661 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:23:23,162 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 19:23:23,162 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 19:23:23,962 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 19:23:23,963 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 19:23:23,964 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 19:23:23,966 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 19:23:24,734 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 19:23:25,735 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 19:23:27,967 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:23:28,365 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:23:32,967 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:23:33,365 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:23:37,968 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:23:38,176 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 19:23:38,176 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 19:23:43,291 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:23:43,367 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:23:46,659 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 19:23:48,368 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:23:48,662 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:23:53,175 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 19:23:53,185 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 19:23:54,295 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:23:56,246 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 19:23:56,248 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 19:23:56,248 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 19:23:56,250 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 19:23:56,769 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 19:23:57,770 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 19:23:58,400 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:24:00,251 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:24:03,401 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:24:05,251 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:24:08,175 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 19:24:08,176 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 19:24:10,312 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:24:13,403 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:24:15,313 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:24:16,662 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 19:24:18,404 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:24:20,676 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:24:23,182 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 19:24:23,184 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 19:24:26,322 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:24:26,911 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 19:24:26,912 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 19:24:26,913 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 19:24:26,914 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 19:24:27,799 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 19:24:28,415 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:24:28,800 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 19:24:31,916 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:24:33,416 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:24:36,916 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:24:38,179 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 19:24:38,180 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 19:24:42,295 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:24:43,417 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:24:46,667 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 19:24:47,669 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:24:48,418 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:24:52,669 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:24:53,179 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 19:24:53,180 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 19:24:58,315 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:24:58,431 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:25:00,896 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 19:25:00,898 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 19:25:00,898 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 19:25:00,899 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 19:25:01,839 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 19:25:02,840 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 19:25:03,430 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:25:03,900 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:25:08,179 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 19:25:08,180 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 19:25:09,311 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:25:13,432 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:25:14,312 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:25:16,671 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 19:25:18,433 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:25:19,673 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:25:23,179 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 19:25:23,180 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 19:25:25,311 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:25:28,440 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:25:30,318 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:25:32,140 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 19:25:32,141 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 19:25:32,141 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 19:25:32,142 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 19:25:32,864 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 19:25:33,536 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:25:34,866 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 19:25:36,143 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:25:38,180 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 19:25:38,180 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 19:25:41,313 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:25:43,663 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:25:46,314 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:25:46,674 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 19:25:48,749 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:25:51,677 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:25:53,180 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 19:25:53,180 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 19:25:57,330 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:25:58,757 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:26:01,656 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 19:26:01,657 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 19:26:01,658 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 19:26:01,659 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 19:26:01,892 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 19:26:02,660 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:26:02,893 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 19:26:04,054 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:26:04,054 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:26:07,660 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:26:08,180 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 19:26:08,180 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 19:26:13,340 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:26:14,056 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:26:14,056 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:26:16,679 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 19:26:18,682 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:26:19,057 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:26:19,057 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:26:23,180 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 19:26:23,180 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 19:26:24,293 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:26:29,069 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:26:29,305 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:26:31,762 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 19:26:31,763 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 19:26:31,764 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 19:26:31,765 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 19:26:31,926 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 19:26:32,927 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 19:26:34,159 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:26:34,766 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:26:38,180 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 19:26:38,180 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 19:26:40,292 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:26:44,160 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:26:44,161 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:26:45,293 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:26:46,682 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 19:26:49,161 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:26:49,162 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:26:50,685 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:26:53,180 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 19:26:53,180 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 19:26:56,293 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:26:59,167 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:27:01,288 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:27:01,492 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 19:27:01,494 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 19:27:01,494 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 19:27:01,496 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 19:27:01,948 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 19:27:02,949 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 19:27:04,168 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:27:06,497 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:27:08,180 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 19:27:08,181 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 19:27:12,286 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:27:14,169 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:27:16,687 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 19:27:17,689 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:27:19,170 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:27:22,690 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:27:23,180 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 19:27:23,181 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 19:27:28,348 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:27:29,171 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:27:33,356 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:27:34,183 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:27:35,868 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 19:27:35,870 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 19:27:35,870 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 19:27:35,871 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 19:27:35,982 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 19:27:36,983 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 19:27:38,180 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 19:27:38,181 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 19:27:39,302 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:27:43,264 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:27:43,264 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:27:44,302 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:27:46,690 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 19:27:48,265 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:27:48,265 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:27:49,693 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:27:53,180 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 19:27:53,181 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 19:27:55,343 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:27:58,266 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:27:58,267 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:28:00,346 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:28:03,273 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:28:05,347 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:28:05,735 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 19:28:05,736 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 19:28:05,736 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 19:28:05,738 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 19:28:06,016 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 19:28:07,017 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 19:28:08,181 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 19:28:08,181 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 19:28:11,265 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:28:13,277 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:28:16,265 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:28:16,694 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 19:28:18,278 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:28:18,278 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:28:21,697 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:28:23,181 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 19:28:23,181 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 19:28:27,318 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:28:28,279 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:28:32,322 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:28:33,290 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:28:33,290 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:28:37,113 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 19:28:37,115 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 19:28:37,115 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 19:28:37,117 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 19:28:38,053 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 19:28:38,118 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:28:38,181 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 19:28:38,181 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 19:28:39,054 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 19:28:43,338 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:28:43,543 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:28:43,543 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:28:46,699 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 19:28:48,544 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:28:48,544 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:28:48,701 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:28:53,181 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 19:28:53,182 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 19:28:54,328 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:28:58,545 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:28:58,546 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:28:59,329 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:29:03,551 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:29:04,337 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:29:05,840 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 19:29:05,842 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 19:29:05,842 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 19:29:05,844 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 19:29:06,081 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 19:29:07,081 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 19:29:08,181 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 19:29:08,182 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 19:29:09,391 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:29:13,552 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:29:14,392 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:29:16,703 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 19:29:18,553 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:29:19,705 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:29:23,181 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 19:29:23,182 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 19:29:25,336 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:29:28,554 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:29:30,342 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:29:33,567 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:29:35,345 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:29:35,732 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 19:29:35,734 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 19:29:35,734 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 19:29:35,735 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 19:29:36,108 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 19:29:37,109 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 19:29:38,182 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 19:29:38,182 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 19:29:40,351 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:29:43,564 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:29:45,352 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:29:46,706 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 19:29:48,565 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:29:50,709 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:29:53,181 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 19:29:53,182 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 19:29:56,302 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:29:58,566 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:30:01,303 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:30:03,571 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:30:06,307 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:30:08,183 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 19:30:08,184 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 19:30:09,227 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 19:30:09,228 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 19:30:09,229 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 19:30:09,230 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 19:30:10,138 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 19:30:11,139 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 19:30:12,231 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:30:13,629 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:30:16,710 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 19:30:17,713 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:30:18,630 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:30:22,713 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:30:23,182 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 19:30:23,182 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 19:30:28,290 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:30:28,631 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:30:28,632 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:30:33,291 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:30:33,633 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:30:33,634 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:30:38,183 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 19:30:38,183 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 19:30:39,273 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:30:40,307 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 19:30:40,308 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 19:30:40,308 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 19:30:40,309 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 19:30:41,174 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 19:30:43,176 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 19:30:43,849 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:30:43,849 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:30:44,310 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:30:46,714 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 19:30:48,850 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:30:49,717 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:30:53,183 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 19:30:53,184 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 19:30:55,316 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:30:58,851 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:31:00,316 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:31:03,857 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:31:05,318 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:31:08,189 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 19:31:08,189 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 19:31:08,724 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 19:31:08,725 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 19:31:08,726 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 19:31:08,727 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 19:31:09,202 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 19:31:10,728 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:31:11,203 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 19:31:14,117 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:31:14,117 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:31:15,729 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:31:16,719 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 19:31:19,118 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:31:19,118 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:31:21,721 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:31:23,189 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 19:31:23,190 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 19:31:27,397 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:31:29,169 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:31:32,398 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:31:34,178 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:31:37,407 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:31:38,193 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 19:31:38,198 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 19:31:40,027 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 19:31:40,029 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 19:31:40,029 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 19:31:40,031 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 19:31:40,227 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 19:31:41,228 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 19:31:43,032 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:31:43,423 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:31:43,424 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:31:46,723 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 19:31:48,424 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:31:48,425 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:31:48,725 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:31:53,193 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 19:31:53,194 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 19:31:54,328 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:31:58,426 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:31:58,426 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:31:59,329 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:32:03,427 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:32:03,427 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:32:04,334 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:32:08,203 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 19:32:08,207 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 19:32:09,367 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:32:10,036 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 19:32:10,037 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 19:32:10,038 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 19:32:10,039 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 19:32:10,259 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 19:32:11,260 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 19:32:13,439 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:32:15,041 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:32:16,725 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 19:32:18,439 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:32:20,727 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:32:23,199 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 19:32:23,200 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 19:32:26,364 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:32:28,441 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:32:28,441 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:32:31,365 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:32:33,442 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:32:33,442 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:32:36,375 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:32:38,209 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 19:32:38,213 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 19:32:41,080 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 19:32:41,082 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 19:32:41,082 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 19:32:41,084 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 19:32:41,289 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 19:32:42,085 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:32:43,290 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 19:32:44,064 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:32:46,729 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 19:32:47,732 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:32:49,064 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:32:52,732 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:32:53,203 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 19:32:53,203 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 19:32:58,328 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:32:59,066 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:32:59,066 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:33:03,328 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:33:04,067 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:33:04,067 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:33:08,206 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 19:33:08,206 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 19:33:09,313 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:33:10,607 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 19:33:10,609 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 19:33:10,609 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 19:33:10,611 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 19:33:11,315 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 19:33:13,316 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 19:33:13,507 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:33:13,507 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:33:14,612 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:33:16,733 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 19:33:18,508 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:33:18,508 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:33:19,736 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:33:23,206 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 19:33:23,207 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 19:33:25,353 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:33:28,510 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:33:30,354 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:33:33,510 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:33:35,355 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:33:38,214 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 19:33:38,215 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 19:33:41,303 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:33:42,032 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 19:33:42,033 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 19:33:42,033 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 19:33:42,035 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 19:33:42,348 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 19:33:43,349 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 19:33:43,749 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:33:43,749 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:33:46,737 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 19:33:46,745 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:33:48,750 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:33:51,743 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:33:53,209 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 19:33:53,210 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 19:33:57,368 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:33:58,751 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:33:58,752 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:34:02,369 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:34:03,752 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:34:03,752 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:34:07,369 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:34:08,222 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 19:34:08,235 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 19:34:13,349 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:34:13,757 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:34:14,349 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 19:34:14,350 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 19:34:14,350 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 19:34:14,352 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 19:34:14,375 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 19:34:15,376 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 19:34:16,741 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 19:34:18,744 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:34:18,758 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:34:23,211 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 19:34:23,212 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 19:34:24,380 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:34:28,760 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:34:29,381 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:34:33,761 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:34:34,382 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:34:38,212 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 19:34:38,212 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 19:34:40,307 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:34:43,770 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:34:44,683 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 19:34:44,686 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 19:34:44,686 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 19:34:44,688 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 19:34:45,404 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 19:34:45,689 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:34:46,745 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 19:34:47,405 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 19:34:49,083 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:34:49,083 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:34:50,748 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:34:53,211 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 19:34:53,212 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 19:34:56,382 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:34:59,085 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:34:59,085 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:35:01,383 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:35:04,086 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:35:04,086 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:35:06,383 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:35:08,215 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 19:35:08,217 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 19:35:12,337 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:35:14,092 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:35:14,140 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 19:35:14,142 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 19:35:14,142 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 19:35:14,143 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 19:35:14,433 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 19:35:15,434 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 19:35:16,749 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 19:35:17,752 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:35:19,092 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:35:22,752 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:35:23,215 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 19:35:23,215 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 19:35:28,343 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:35:29,094 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:35:33,344 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:35:34,094 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:35:38,215 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 19:35:38,216 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 19:35:39,300 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:35:44,101 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:35:44,312 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:35:46,719 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 19:35:46,721 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 19:35:46,721 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 19:35:46,723 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 19:35:46,753 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 19:35:47,457 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 19:35:49,113 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:35:49,459 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 19:35:49,756 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:35:53,215 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 19:35:53,216 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 19:35:55,346 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:35:59,114 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:35:59,115 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:36:00,347 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:36:04,115 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:36:04,115 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:36:05,347 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:36:08,215 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 19:36:08,216 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 19:36:10,371 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:36:14,123 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:36:15,383 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:36:16,760 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 19:36:17,134 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 19:36:17,136 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 19:36:17,136 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 19:36:17,137 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 19:36:17,483 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 19:36:19,122 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:36:19,484 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 19:36:21,138 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:36:23,215 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 19:36:23,216 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 19:36:26,307 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:36:28,282 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:36:31,308 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:36:33,464 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:36:36,308 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:36:38,215 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 19:36:38,216 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 19:36:41,378 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:36:43,474 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:36:46,376 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:36:46,761 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 19:36:47,465 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 19:36:47,466 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 19:36:47,467 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 19:36:47,468 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 19:36:47,512 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 19:36:48,611 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:36:49,514 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 19:36:51,469 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:36:53,216 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 19:36:53,217 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 19:36:57,305 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:36:58,613 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:37:02,306 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:37:03,614 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:37:07,307 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:37:08,216 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 19:37:08,217 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 19:37:12,340 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:37:13,770 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:37:16,771 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 19:37:17,771 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:37:18,778 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:37:19,628 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 19:37:19,629 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 19:37:19,630 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 19:37:19,631 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 19:37:20,539 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 19:37:21,540 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 19:37:23,216 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 19:37:23,216 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 19:37:23,294 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:37:28,295 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:37:28,781 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:37:33,295 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:37:33,782 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:37:38,216 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 19:37:38,216 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 19:37:38,339 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:37:43,340 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:37:43,791 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:37:46,772 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 19:37:48,775 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:37:48,792 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:37:49,592 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 19:37:49,593 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 19:37:49,594 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 19:37:49,597 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 19:37:50,568 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 19:37:51,569 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 19:37:53,216 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 19:37:53,217 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 19:37:54,293 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:37:58,995 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:37:59,294 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:38:03,996 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:38:04,294 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:38:08,216 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 19:38:08,217 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 19:38:09,343 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:38:14,007 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:38:14,352 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:38:16,776 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 19:38:19,006 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:38:19,440 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 19:38:19,441 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 19:38:19,441 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 19:38:19,442 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:38:19,443 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 19:38:19,605 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 19:38:21,607 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 19:38:23,216 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 19:38:23,217 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 19:38:25,381 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:38:29,007 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:38:30,382 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:38:34,008 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:38:35,383 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:38:38,216 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 19:38:38,217 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 19:38:41,359 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:38:44,010 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:38:46,366 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:38:46,777 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 19:38:49,021 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:38:51,781 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:38:52,272 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 19:38:52,274 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 19:38:52,275 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 19:38:52,277 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 19:38:52,633 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 19:38:53,219 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 19:38:53,220 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 19:38:53,634 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 19:38:57,356 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:38:59,020 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:39:02,357 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:39:04,021 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:39:07,357 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:39:08,219 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 19:39:08,219 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 19:39:13,265 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:39:13,323 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:39:16,782 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 19:39:18,273 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:39:18,791 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:39:22,906 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 19:39:22,908 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 19:39:22,908 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 19:39:22,911 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 19:39:23,285 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:39:23,285 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 19:39:23,286 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 19:39:23,661 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 19:39:24,457 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:39:25,662 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 19:39:28,286 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:39:29,457 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:39:33,286 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:39:34,458 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:39:38,285 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 19:39:38,286 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 19:39:40,426 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:39:43,329 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:39:45,426 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:39:46,786 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 19:39:48,340 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:39:50,795 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:39:53,290 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 19:39:53,290 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 19:39:54,178 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 19:39:54,179 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 19:39:54,179 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 19:39:54,184 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 19:39:54,694 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 19:39:55,695 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 19:39:56,185 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:39:58,580 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:40:01,185 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:40:03,581 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:40:06,186 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:40:08,290 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 19:40:08,291 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 19:40:11,396 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:40:13,583 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:40:16,397 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:40:16,789 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 19:40:18,592 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:40:21,799 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:40:23,291 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 19:40:23,291 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 19:40:24,034 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 19:40:24,036 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 19:40:24,036 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 19:40:24,038 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 19:40:24,723 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 19:40:25,724 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 19:40:27,039 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:40:28,592 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:40:32,039 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:40:33,592 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:40:37,040 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:40:38,446 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 19:40:38,446 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 19:40:42,592 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:40:43,594 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:40:46,793 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 19:40:47,796 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:40:48,594 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:40:52,802 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:40:53,357 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 19:40:53,358 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 19:40:57,426 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 19:40:57,428 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 19:40:57,428 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 19:40:57,433 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 19:40:57,752 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 19:40:58,434 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:40:58,821 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:40:59,754 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 19:41:03,434 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:41:03,821 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:41:08,352 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 19:41:08,353 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 19:41:08,465 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:41:13,466 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:41:13,920 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:41:16,797 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 19:41:18,799 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:41:18,920 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:41:23,357 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 19:41:23,358 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 19:41:24,453 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:41:28,930 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:41:29,315 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 19:41:29,317 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 19:41:29,317 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 19:41:29,321 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 19:41:29,786 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 19:41:30,322 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:41:31,788 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 19:41:33,931 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:41:35,322 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:41:38,354 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 19:41:38,355 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 19:41:40,466 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:41:43,940 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:41:45,466 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:41:46,800 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 19:41:48,940 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:41:50,802 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:41:53,354 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 19:41:53,355 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 19:41:56,493 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:41:58,954 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:42:00,640 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 19:42:00,642 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 19:42:00,643 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 19:42:00,648 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 19:42:00,818 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 19:42:01,649 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:42:01,819 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 19:42:04,044 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:42:06,650 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:42:08,354 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 19:42:08,355 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 19:42:12,475 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:42:14,046 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:42:16,803 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 19:42:17,805 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:42:19,046 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:42:22,806 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:42:23,354 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 19:42:23,355 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 19:42:28,446 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:42:29,056 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:42:31,640 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 19:42:31,642 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 19:42:31,643 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 19:42:31,648 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 19:42:31,845 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 19:42:33,649 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:42:33,847 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 19:42:34,053 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:42:38,355 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 19:42:38,355 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 19:42:39,497 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:42:44,101 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:42:44,498 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:42:46,807 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 19:42:49,102 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:42:49,809 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:42:53,355 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 19:42:53,355 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 19:42:55,524 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:42:59,111 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:43:00,532 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:43:02,003 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 19:43:02,004 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 19:43:02,005 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 19:43:02,010 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 19:43:02,876 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 19:43:03,877 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 19:43:04,153 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:43:06,011 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:43:08,355 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 19:43:08,356 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 19:43:11,478 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:43:13,400 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:43:16,478 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:43:16,811 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 19:43:18,401 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:43:21,813 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:43:23,355 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 19:43:23,355 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 19:43:27,519 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:43:28,409 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:43:31,739 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 19:43:31,740 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 19:43:31,741 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 19:43:31,743 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 19:43:31,901 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 19:43:32,744 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:43:33,407 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:43:33,902 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 19:43:37,745 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:43:38,355 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 19:43:38,355 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 19:43:43,408 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:43:43,501 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:43:46,814 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 19:43:48,473 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:43:48,816 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:43:53,355 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 19:43:53,356 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 19:43:54,455 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:43:58,474 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:43:59,455 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:44:03,486 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:44:04,461 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:44:06,936 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 19:44:06,937 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 19:44:06,937 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 19:44:06,940 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 19:44:07,932 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 19:44:07,933 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 19:44:08,355 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 19:44:08,356 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 19:44:10,448 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:44:13,484 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:44:15,449 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:44:16,818 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 19:44:18,484 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:44:20,820 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:44:23,355 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 19:44:23,356 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 19:44:26,458 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:44:28,486 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:44:31,464 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:44:33,493 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:44:36,463 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:44:37,393 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 19:44:37,394 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 19:44:37,394 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 19:44:37,399 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 19:44:37,963 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 19:44:38,356 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 19:44:38,356 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 19:44:39,964 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 19:44:41,472 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:44:43,540 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:44:46,472 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:44:46,822 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 19:44:48,541 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:44:51,824 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:44:53,356 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 19:44:53,356 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 19:44:57,490 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:44:58,466 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:45:02,498 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:45:03,481 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:45:05,834 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 19:45:05,836 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 19:45:05,837 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 19:45:05,838 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 19:45:05,992 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 19:45:07,839 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:45:07,994 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 19:45:08,356 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 19:45:08,356 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 19:45:13,476 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:45:13,493 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:45:16,827 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 19:45:18,477 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:45:18,829 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:45:23,356 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 19:45:23,357 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 19:45:24,429 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:45:28,484 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:45:29,435 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:45:33,484 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:45:34,268 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 19:45:34,270 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 19:45:34,270 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 19:45:34,271 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 19:45:35,022 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 19:45:35,272 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:45:36,027 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 19:45:38,356 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 19:45:38,357 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 19:45:40,441 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:45:43,670 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:45:45,442 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:45:46,830 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 19:45:48,671 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:45:48,671 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:45:50,832 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:45:53,356 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 19:45:53,357 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 19:45:56,524 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:45:58,672 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:45:58,672 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:46:01,531 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:46:03,681 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:46:03,682 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:46:05,282 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 19:46:05,283 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 19:46:05,284 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 19:46:05,285 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 19:46:06,050 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 19:46:06,051 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 19:46:07,286 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:46:08,356 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 19:46:08,356 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 19:46:12,441 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:46:13,682 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:46:16,833 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 19:46:17,836 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:46:18,683 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:46:22,837 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:46:23,356 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 19:46:23,357 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 19:46:28,469 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:46:28,684 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:46:33,482 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:46:33,693 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:46:35,059 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 19:46:35,060 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 19:46:35,060 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 19:46:35,062 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 19:46:35,075 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 19:46:36,076 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 19:46:38,356 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 19:46:38,357 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 19:46:38,500 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:46:43,501 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:46:43,692 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:46:46,837 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 19:46:48,693 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:46:48,840 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:46:53,357 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 19:46:53,357 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 19:46:54,494 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:46:58,694 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:46:59,494 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:47:03,703 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:47:04,501 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:47:06,921 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 19:47:06,924 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 19:47:06,924 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 19:47:06,926 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 19:47:07,107 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 19:47:08,108 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 19:47:08,357 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 19:47:08,357 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 19:47:10,463 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:47:13,697 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:47:15,463 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:47:16,841 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 19:47:18,697 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:47:20,844 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:47:23,357 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 19:47:23,358 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 19:47:26,472 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:47:28,699 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:47:31,472 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:47:33,710 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:47:36,479 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:47:37,720 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 19:47:37,722 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 19:47:37,722 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 19:47:37,723 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 19:47:38,144 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 19:47:38,144 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 19:47:38,357 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 19:47:38,358 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 19:47:41,509 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:47:43,710 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:47:46,509 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:47:46,845 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 19:47:48,710 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:47:51,848 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:47:53,357 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 19:47:53,358 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 19:47:57,492 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:47:58,727 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:48:02,492 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:48:03,728 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:48:07,499 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:48:08,358 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 19:48:08,358 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 19:48:13,104 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 19:48:13,106 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 19:48:13,106 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 19:48:13,106 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:48:13,107 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 19:48:13,175 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 19:48:13,731 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:48:15,177 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 19:48:16,850 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 19:48:18,732 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:48:18,852 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:48:23,358 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 19:48:23,359 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 19:48:24,513 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:48:28,733 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:48:29,514 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:48:33,734 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:48:34,515 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:48:38,358 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 19:48:38,359 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 19:48:39,580 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:48:43,745 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:48:44,585 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:48:44,921 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 19:48:44,922 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 19:48:44,922 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 19:48:44,923 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 19:48:45,206 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 19:48:46,854 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 19:48:47,207 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 19:48:48,742 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:48:49,856 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:48:53,359 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 19:48:53,359 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 19:48:55,450 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:48:58,743 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:49:00,451 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:49:03,808 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:49:05,451 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:49:08,359 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 19:49:08,359 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 19:49:10,483 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:49:13,818 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:49:15,488 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:49:16,858 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 19:49:18,180 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 19:49:18,182 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 19:49:18,182 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 19:49:18,184 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 19:49:18,235 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 19:49:18,817 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:49:19,236 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 19:49:21,185 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:49:23,359 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 19:49:23,359 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 19:49:26,518 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:49:28,818 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:49:31,518 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:49:33,830 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:49:36,519 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:49:38,359 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 19:49:38,359 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 19:49:41,521 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:49:43,838 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:49:46,526 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:49:46,859 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 19:49:47,631 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 19:49:47,633 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 19:49:47,633 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 19:49:47,638 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 19:49:48,268 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 19:49:49,030 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:49:49,030 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:49:49,269 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 19:49:51,640 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:49:53,359 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 19:49:53,359 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 19:49:57,472 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:49:59,031 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:50:02,472 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:50:04,032 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:50:07,474 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:50:08,359 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 19:50:08,359 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 19:50:12,512 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:50:14,043 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:50:16,867 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 19:50:17,868 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:50:17,946 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 19:50:17,948 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 19:50:17,948 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 19:50:17,953 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 19:50:18,298 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 19:50:19,299 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 19:50:19,345 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:50:19,345 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:50:22,954 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:50:23,359 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 19:50:23,360 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 19:50:28,474 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:50:29,396 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:50:29,490 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:50:33,475 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:50:34,491 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:50:38,359 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 19:50:38,360 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 19:50:38,496 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:50:43,496 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:50:43,503 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:50:46,866 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 19:50:47,195 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 19:50:47,197 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 19:50:47,198 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 19:50:47,199 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 19:50:47,325 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 19:50:48,593 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:50:48,593 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:50:49,201 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:50:49,326 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 19:50:53,359 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 19:50:53,360 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 19:50:54,451 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:50:58,708 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:50:58,801 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:50:59,452 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:51:03,802 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:51:04,452 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:51:08,359 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 19:51:08,360 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 19:51:09,481 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:51:13,820 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:51:14,493 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:51:15,323 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 19:51:15,325 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 19:51:15,325 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 19:51:15,327 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 19:51:15,355 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 19:51:16,869 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 19:51:17,356 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 19:51:18,814 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:51:18,815 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:51:19,872 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:51:23,359 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 19:51:23,360 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 19:51:25,461 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:51:28,816 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:51:28,816 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:51:30,462 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:51:33,817 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:51:33,817 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:51:35,462 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:51:38,360 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 19:51:38,360 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 19:51:40,534 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:51:43,829 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:51:45,055 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 19:51:45,057 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 19:51:45,057 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 19:51:45,062 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 19:51:45,387 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 19:51:46,063 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:51:46,873 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 19:51:47,389 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 19:51:48,828 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:51:48,828 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:51:51,876 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:51:53,360 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 19:51:53,361 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 19:51:57,517 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:51:58,830 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:51:58,830 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:52:02,517 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:52:03,831 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:52:03,831 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:52:07,518 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:52:08,366 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 19:52:08,373 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 19:52:13,477 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:52:13,839 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:52:14,273 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 19:52:14,274 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 19:52:14,274 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 19:52:14,275 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 19:52:14,413 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 19:52:15,414 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 19:52:16,878 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 19:52:18,840 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:52:18,880 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:52:23,362 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 19:52:23,363 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 19:52:24,516 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:52:28,841 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:52:29,516 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:52:33,842 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:52:34,517 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:52:38,362 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 19:52:38,363 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 19:52:40,505 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:52:43,852 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:52:45,089 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 19:52:45,091 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 19:52:45,091 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 19:52:45,092 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 19:52:45,443 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 19:52:46,093 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:52:46,882 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 19:52:47,444 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 19:52:48,850 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:52:51,884 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:52:53,362 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 19:52:53,363 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 19:52:57,458 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:52:58,852 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:53:02,458 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:53:03,853 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:53:07,459 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:53:08,362 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 19:53:08,363 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 19:53:13,439 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:53:13,864 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:53:14,457 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 19:53:14,458 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 19:53:14,459 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 19:53:14,460 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 19:53:14,475 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 19:53:15,476 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 19:53:16,886 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 19:53:18,884 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:53:18,884 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:53:18,889 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:53:23,362 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 19:53:23,363 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 19:53:24,483 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:53:28,900 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:53:28,900 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:53:29,484 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:53:33,901 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:53:33,901 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:53:34,484 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:53:38,363 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 19:53:38,363 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 19:53:39,512 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:53:43,909 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:53:44,510 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:53:45,111 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 19:53:45,112 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 19:53:45,112 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 19:53:45,114 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 19:53:45,499 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 19:53:46,890 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 19:53:47,501 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 19:53:48,908 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:53:48,908 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:53:49,892 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:53:53,363 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 19:53:53,363 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 19:53:55,531 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:53:58,910 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:53:58,910 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:54:00,532 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:54:03,910 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:54:03,911 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:54:05,532 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:54:08,363 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 19:54:08,364 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 19:54:11,510 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:54:13,918 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:54:15,184 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 19:54:15,186 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 19:54:15,186 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 19:54:15,188 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 19:54:15,535 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 19:54:16,893 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 19:54:16,894 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:54:17,537 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 19:54:18,915 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:54:21,895 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:54:23,363 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 19:54:23,364 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 19:54:27,515 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:54:28,917 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:54:32,516 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:54:34,024 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:54:34,024 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:54:37,517 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:54:38,364 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 19:54:38,364 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 19:54:43,459 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:54:44,028 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:54:46,898 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 19:54:48,899 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:54:49,031 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:54:49,236 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 19:54:49,238 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 19:54:49,238 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 19:54:49,239 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 19:54:49,568 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 19:54:51,570 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 19:54:53,364 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 19:54:53,365 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 19:54:54,445 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:54:59,033 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:54:59,446 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:55:04,034 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:55:04,447 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:55:08,364 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 19:55:08,364 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 19:55:09,457 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:55:14,038 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:55:14,461 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:55:16,902 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 19:55:18,419 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 19:55:18,420 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 19:55:18,421 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 19:55:18,422 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 19:55:18,606 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 19:55:19,042 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:55:19,607 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 19:55:20,423 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:55:23,364 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 19:55:23,365 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 19:55:25,483 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:55:29,044 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:55:30,484 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:55:34,044 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:55:35,484 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:55:38,364 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 19:55:38,365 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 19:55:40,530 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:55:44,050 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:55:45,538 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:55:46,902 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 19:55:48,508 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 19:55:48,509 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 19:55:48,509 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 19:55:48,510 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 19:55:48,632 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 19:55:49,052 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:55:49,632 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 19:55:51,511 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:55:53,364 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 19:55:53,365 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 19:55:57,479 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:55:59,054 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:56:02,480 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:56:04,055 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:56:07,480 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:56:08,364 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 19:56:08,365 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 19:56:12,584 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:56:14,064 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:56:16,908 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 19:56:17,909 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:56:18,337 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 19:56:18,338 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 19:56:18,339 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 19:56:18,340 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 19:56:18,669 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 19:56:19,067 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:56:19,068 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:56:19,670 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 19:56:23,341 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:56:23,364 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 19:56:23,365 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 19:56:28,532 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:56:29,069 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:56:29,069 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:56:33,533 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:56:34,070 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:56:34,070 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:56:38,364 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 19:56:38,365 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 19:56:39,486 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:56:44,071 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:56:44,071 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:56:44,494 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:56:46,914 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 19:56:49,079 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:56:49,916 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:56:50,131 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 19:56:50,133 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 19:56:50,134 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 19:56:50,136 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 19:56:50,696 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 19:56:51,697 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 19:56:53,365 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 19:56:53,365 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 19:56:55,534 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:56:59,079 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:57:00,535 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:57:04,080 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:57:05,535 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:57:08,365 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 19:57:08,365 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 19:57:11,444 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:57:14,081 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:57:16,450 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:57:16,916 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 19:57:19,087 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:57:19,154 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 19:57:19,155 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 19:57:19,156 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 19:57:19,162 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 19:57:19,722 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 19:57:21,723 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 19:57:22,163 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:57:23,365 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 19:57:23,365 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 19:57:27,529 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:57:29,089 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:57:32,530 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:57:34,090 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:57:37,531 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:57:38,365 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 19:57:38,365 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 19:57:43,530 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:57:44,099 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:57:46,920 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 19:57:48,426 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 19:57:48,427 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 19:57:48,428 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 19:57:48,429 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 19:57:48,744 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 19:57:49,099 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:57:49,100 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:57:49,430 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:57:49,745 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 19:57:53,365 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 19:57:53,366 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 19:57:54,475 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:57:59,107 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:57:59,107 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:57:59,476 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:58:04,108 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:58:04,108 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:58:04,476 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:58:08,365 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 19:58:08,366 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 19:58:09,530 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:58:14,109 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:58:14,110 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:58:14,531 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:58:16,924 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 19:58:19,119 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:58:19,928 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:58:22,255 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 19:58:22,257 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 19:58:22,257 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 19:58:22,260 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 19:58:22,775 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 19:58:23,365 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 19:58:23,366 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 19:58:23,776 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 19:58:25,531 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:58:29,118 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:58:30,531 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:58:34,118 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:58:34,119 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:58:35,532 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:58:38,366 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 19:58:38,366 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 19:58:41,447 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:58:44,245 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:58:44,245 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:58:46,448 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:58:46,927 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 19:58:49,251 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:58:51,940 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:58:53,368 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 19:58:53,372 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 19:58:53,669 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 19:58:53,670 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 19:58:53,671 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 19:58:53,673 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 19:58:53,801 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 19:58:55,802 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 19:58:57,674 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:58:59,251 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:59:02,675 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:59:04,252 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:59:07,676 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:59:08,368 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 19:59:08,368 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 19:59:13,443 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:59:14,276 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:59:16,930 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 19:59:18,941 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:59:19,283 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:59:23,368 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 19:59:23,368 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 19:59:23,995 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 19:59:23,996 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 19:59:23,997 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 19:59:23,997 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:59:24,002 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 19:59:24,828 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 19:59:25,829 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 19:59:28,390 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:59:28,390 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:59:29,003 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:59:33,391 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:59:33,391 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:59:34,004 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:59:38,369 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 19:59:38,369 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 19:59:39,496 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:59:43,393 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:59:43,393 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:59:44,497 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:59:46,934 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 19:59:48,402 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 19:59:49,945 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:59:53,369 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 19:59:53,371 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 19:59:53,855 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 19:59:53,857 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 19:59:53,857 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 19:59:53,859 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 19:59:54,854 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 19:59:55,855 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 19:59:55,860 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 19:59:58,402 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:00:00,860 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:00:03,403 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:00:05,861 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:00:08,369 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 20:00:08,370 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 20:00:11,522 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:00:13,545 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:00:13,545 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:00:16,523 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:00:16,937 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 20:00:18,562 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:00:21,945 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:00:23,369 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 20:00:23,369 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 20:00:24,348 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 20:00:24,350 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 20:00:24,350 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 20:00:24,351 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 20:00:24,884 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 20:00:25,885 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 20:00:27,352 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:00:28,746 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:00:28,746 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:00:32,353 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:00:33,747 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:00:33,747 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:00:37,353 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:00:38,369 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 20:00:38,370 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 20:00:42,457 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:00:43,795 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:00:43,795 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:00:46,942 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 20:00:47,944 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:00:48,802 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:00:52,954 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:00:53,207 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 20:00:53,208 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 20:00:53,209 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 20:00:53,210 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 20:00:53,608 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 20:00:53,609 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 20:00:53,913 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 20:00:53,914 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 20:00:58,701 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:00:58,805 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:01:03,702 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:01:03,806 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:01:08,608 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 20:01:08,609 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 20:01:08,754 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:01:13,754 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:01:13,807 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:01:16,946 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 20:01:18,813 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:01:18,951 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:01:23,482 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 20:01:23,484 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 20:01:23,485 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 20:01:23,487 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 20:01:23,880 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:01:23,881 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:01:23,883 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 20:01:23,884 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 20:01:23,946 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 20:01:24,027 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:01:25,948 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 20:01:29,028 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:01:29,882 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:01:29,882 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:01:34,028 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:01:35,112 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:01:35,113 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:01:38,632 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 20:01:38,633 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 20:01:39,747 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:01:44,114 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:01:44,747 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:01:46,959 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 20:01:49,122 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:01:49,960 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:01:52,315 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 20:01:52,317 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 20:01:52,317 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 20:01:52,319 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 20:01:52,982 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 20:01:53,632 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 20:01:53,633 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 20:01:53,983 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 20:01:55,746 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:01:59,119 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:02:00,747 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:02:04,120 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:02:05,748 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:02:08,632 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 20:02:08,633 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 20:02:10,753 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:02:14,122 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:02:15,754 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:02:16,956 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 20:02:19,129 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:02:20,967 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:02:23,636 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 20:02:23,636 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 20:02:25,130 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 20:02:25,132 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 20:02:25,132 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 20:02:25,137 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 20:02:26,014 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 20:02:26,014 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 20:02:26,138 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:02:29,135 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:02:31,139 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:02:34,135 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:02:36,140 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:02:38,666 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 20:02:38,667 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 20:02:41,773 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:02:44,137 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:02:46,774 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:02:46,960 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 20:02:49,138 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:02:51,964 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:02:53,672 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 20:02:53,673 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 20:02:56,615 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 20:02:56,616 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 20:02:56,617 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 20:02:56,621 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 20:02:57,037 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 20:02:57,622 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:02:58,038 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 20:02:59,147 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:03:02,623 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:03:04,147 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:03:07,623 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:03:08,669 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 20:03:08,669 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 20:03:12,791 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:03:14,233 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:03:16,964 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 20:03:17,966 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:03:19,234 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:03:22,967 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:03:23,673 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 20:03:23,674 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 20:03:28,843 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:03:29,244 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:03:29,767 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 20:03:29,768 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 20:03:29,768 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 20:03:29,774 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 20:03:30,064 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 20:03:32,065 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 20:03:34,245 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:03:34,775 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:03:38,672 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 20:03:38,673 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 20:03:40,763 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:03:44,274 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:03:45,764 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:03:46,968 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 20:03:49,275 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:03:50,970 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:03:53,673 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 20:03:53,674 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 20:03:56,799 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:03:57,394 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 20:03:57,396 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 20:03:57,397 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 20:03:57,398 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 20:03:58,085 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 20:03:58,085 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 20:03:58,800 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:04:02,399 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:04:03,800 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:04:07,399 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:04:08,673 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 20:04:08,674 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 20:04:12,743 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:04:13,876 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:04:16,973 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 20:04:17,975 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:04:18,876 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:04:22,977 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:04:23,686 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 20:04:23,687 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 20:04:28,026 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 20:04:28,028 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 20:04:28,029 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 20:04:28,029 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:04:28,030 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 20:04:28,114 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 20:04:28,881 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:04:30,115 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 20:04:33,031 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:04:33,882 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:04:38,032 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:04:38,683 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 20:04:38,684 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 20:04:43,763 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:04:43,912 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:04:46,977 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 20:04:48,913 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:04:48,979 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:04:53,696 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 20:04:53,700 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 20:04:54,775 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:04:58,924 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:04:59,031 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 20:04:59,033 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 20:04:59,033 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 20:04:59,035 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 20:04:59,138 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 20:05:00,036 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:05:00,139 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 20:05:03,924 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:05:05,036 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:05:08,687 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 20:05:08,688 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 20:05:10,786 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:05:13,926 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:05:15,787 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:05:16,980 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 20:05:18,926 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:05:20,983 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:05:23,693 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 20:05:23,698 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 20:05:26,795 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:05:28,935 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:05:29,581 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 20:05:29,582 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 20:05:29,583 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 20:05:29,585 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 20:05:30,168 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 20:05:30,168 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 20:05:32,586 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:05:33,980 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:05:37,586 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:05:38,692 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 20:05:38,693 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 20:05:42,853 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:05:43,783 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:05:46,983 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 20:05:47,985 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:05:48,784 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:05:52,986 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:05:53,693 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 20:05:53,693 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 20:05:58,790 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:05:58,799 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:05:59,982 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 20:05:59,984 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 20:05:59,985 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 20:05:59,986 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 20:06:00,199 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 20:06:02,201 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 20:06:03,791 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:06:03,987 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:06:08,692 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 20:06:08,693 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 20:06:09,836 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:06:13,841 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:06:14,837 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:06:16,989 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 20:06:18,926 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:06:19,991 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:06:23,693 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 20:06:23,693 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 20:06:25,853 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:06:28,935 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:06:30,857 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:06:31,735 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 20:06:31,736 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 20:06:31,736 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 20:06:31,738 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 20:06:32,227 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 20:06:32,228 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 20:06:34,131 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:06:36,739 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:06:38,692 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 20:06:38,693 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 20:06:41,839 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:06:44,133 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:06:46,840 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:06:46,992 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 20:06:49,133 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:06:51,995 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:06:53,693 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 20:06:53,693 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 20:06:57,797 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:06:59,149 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:07:02,168 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 20:07:02,171 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 20:07:02,171 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 20:07:02,173 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 20:07:02,254 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 20:07:03,174 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:07:04,148 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:07:04,256 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 20:07:08,174 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:07:08,693 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 20:07:08,693 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 20:07:13,766 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:07:14,149 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:07:16,996 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 20:07:18,998 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:07:19,247 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:07:19,248 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:07:23,693 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 20:07:23,693 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 20:07:24,824 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:07:29,252 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:07:29,834 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:07:34,256 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:07:34,256 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:07:34,835 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:07:34,994 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 20:07:34,995 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 20:07:34,996 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 20:07:34,997 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 20:07:35,283 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 20:07:36,284 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 20:07:38,693 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 20:07:38,694 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 20:07:40,811 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:07:44,390 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:07:44,391 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:07:45,811 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:07:46,999 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 20:07:49,529 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:07:51,002 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:07:53,693 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 20:07:53,694 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 20:07:56,788 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:07:59,530 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:07:59,530 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:08:01,796 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:08:04,534 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:08:06,338 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 20:08:06,340 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 20:08:06,340 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 20:08:06,342 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 20:08:07,314 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 20:08:07,343 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:08:08,315 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 20:08:08,693 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 20:08:08,694 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 20:08:12,852 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:08:13,736 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:08:13,737 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:08:17,003 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 20:08:18,005 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:08:18,737 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:08:18,737 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:08:23,006 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:08:23,693 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 20:08:23,694 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 20:08:28,739 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:08:28,739 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:08:28,766 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:08:33,745 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:08:33,771 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:08:38,698 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 20:08:38,698 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 20:08:38,815 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:08:39,747 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 20:08:39,749 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 20:08:39,750 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 20:08:39,753 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 20:08:40,342 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 20:08:40,342 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 20:08:43,747 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:08:44,754 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:08:47,008 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 20:08:48,748 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:08:50,010 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:08:53,698 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 20:08:53,698 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 20:08:55,836 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:08:58,749 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:09:00,836 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:09:03,753 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:09:05,844 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:09:06,962 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 20:09:06,964 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 20:09:06,965 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 20:09:06,966 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 20:09:07,365 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 20:09:08,366 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 20:09:08,698 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 20:09:08,699 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 20:09:11,789 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:09:13,758 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:09:16,790 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:09:17,012 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 20:09:18,759 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:09:22,014 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:09:23,698 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 20:09:23,699 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 20:09:27,772 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:09:28,760 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:09:32,778 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:09:33,771 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:09:35,565 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 20:09:35,567 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 20:09:35,567 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 20:09:35,569 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 20:09:36,398 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 20:09:36,399 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 20:09:38,570 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:09:38,698 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 20:09:38,699 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 20:09:43,848 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:09:43,962 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:09:43,962 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:09:47,017 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 20:09:48,963 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:09:48,963 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:09:49,019 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:09:53,698 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 20:09:53,699 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 20:09:54,763 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:09:58,964 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:09:58,965 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:09:59,764 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:10:03,975 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:10:04,770 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:10:06,115 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 20:10:06,117 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 20:10:06,117 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 20:10:06,118 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 20:10:06,428 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 20:10:08,430 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 20:10:08,699 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 20:10:08,699 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 20:10:09,785 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:10:13,971 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:10:13,972 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:10:14,786 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:10:17,020 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 20:10:18,972 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:10:18,972 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:10:20,023 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:10:23,699 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 20:10:23,699 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 20:10:25,861 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:10:28,974 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:10:28,974 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:10:30,861 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:10:33,983 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:10:35,868 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:10:37,905 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 20:10:37,907 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 20:10:37,907 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 20:10:37,909 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 20:10:38,463 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 20:10:38,463 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 20:10:38,699 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 20:10:38,700 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 20:10:41,854 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:10:44,051 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:10:46,855 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:10:47,024 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 20:10:49,052 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:10:52,027 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:10:53,699 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 20:10:53,699 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 20:10:57,814 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:10:59,053 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:11:02,822 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:11:04,058 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:11:07,824 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:11:08,278 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 20:11:08,280 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 20:11:08,281 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 20:11:08,282 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 20:11:08,494 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 20:11:08,699 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 20:11:08,700 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 20:11:11,496 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 20:11:13,815 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:11:14,061 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:11:17,028 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 20:11:19,031 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:11:19,062 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:11:23,699 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 20:11:23,700 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 20:11:24,793 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:11:29,075 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:11:29,075 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:11:29,793 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:11:34,081 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:11:34,801 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:11:38,699 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 20:11:38,700 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 20:11:38,952 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 20:11:38,954 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 20:11:38,954 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 20:11:38,956 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 20:11:39,526 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 20:11:39,957 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:11:41,528 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 20:11:44,359 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:11:44,359 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:11:44,957 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:11:47,032 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 20:11:49,360 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:11:50,035 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:11:53,880 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 20:11:53,881 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 20:11:55,038 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:11:59,361 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:12:00,039 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:12:04,361 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:12:05,039 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:12:08,803 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 20:12:08,804 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 20:12:10,910 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:12:13,009 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 20:12:13,010 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 20:12:13,011 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 20:12:13,013 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 20:12:13,553 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 20:12:14,414 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:12:15,555 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 20:12:16,014 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:12:17,037 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 20:12:19,415 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:12:19,415 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:12:21,039 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:12:23,799 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 20:12:23,800 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 20:12:26,965 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:12:29,555 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:12:29,556 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:12:31,966 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:12:34,556 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:12:34,557 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:12:36,966 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:12:38,813 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 20:12:38,816 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 20:12:42,914 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:12:44,412 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 20:12:44,413 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 20:12:44,413 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 20:12:44,414 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 20:12:44,582 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 20:12:44,805 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:12:44,806 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:12:47,040 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 20:12:47,584 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 20:12:48,043 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:12:49,806 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:12:53,043 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:12:53,804 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 20:12:53,805 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 20:12:58,808 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:12:58,933 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:13:03,808 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:13:03,809 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:13:03,933 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:13:08,805 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 20:13:08,808 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 20:13:08,939 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:13:13,819 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:13:13,819 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:13:13,942 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:13:14,192 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 20:13:14,194 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 20:13:14,194 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 20:13:14,195 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 20:13:14,612 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 20:13:15,613 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 20:13:17,044 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 20:13:18,820 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:13:18,821 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:13:19,047 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:13:23,805 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 20:13:23,806 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 20:13:24,893 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:13:28,822 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:13:28,822 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:13:29,894 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:13:33,823 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:13:33,823 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:13:34,894 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:13:38,810 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 20:13:38,811 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 20:13:40,896 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:13:43,828 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:13:44,358 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 20:13:44,359 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 20:13:44,360 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 20:13:44,361 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 20:13:44,644 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 20:13:46,362 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:13:47,048 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 20:13:47,647 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 20:13:48,829 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:13:52,051 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:13:53,808 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 20:13:53,808 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 20:13:57,942 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:13:59,018 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:13:59,018 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:14:02,942 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:14:04,018 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:14:04,019 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:14:07,948 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:14:08,826 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 20:14:08,826 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 20:14:12,985 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:14:14,026 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:14:14,052 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 20:14:14,053 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 20:14:14,053 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 20:14:14,055 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 20:14:14,676 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 20:14:15,676 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 20:14:17,053 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 20:14:18,055 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:14:19,028 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:14:23,055 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:14:23,810 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 20:14:23,811 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 20:14:28,896 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:14:29,085 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:14:33,896 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:14:34,086 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:14:34,086 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:14:38,810 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 20:14:38,811 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 20:14:38,964 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:14:43,970 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:14:44,089 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:14:47,055 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 20:14:47,765 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 20:14:47,769 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 20:14:47,769 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 20:14:47,771 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 20:14:48,702 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 20:14:49,157 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:14:49,703 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 20:14:49,772 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:14:53,810 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 20:14:53,811 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 20:14:54,896 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:14:59,220 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:14:59,897 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:15:04,221 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:15:04,221 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:15:04,897 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:15:08,811 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 20:15:08,811 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 20:15:09,925 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:15:14,229 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:15:14,935 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:15:16,930 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 20:15:16,932 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 20:15:16,932 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 20:15:16,934 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 20:15:17,059 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 20:15:17,729 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 20:15:19,353 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:15:19,731 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 20:15:20,062 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:15:23,811 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 20:15:23,811 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 20:15:25,941 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:15:29,355 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:15:30,943 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:15:34,355 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:15:35,944 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:15:38,811 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 20:15:38,811 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 20:15:41,914 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:15:44,363 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:15:46,348 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 20:15:46,349 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 20:15:46,350 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 20:15:46,351 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 20:15:46,758 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 20:15:47,062 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 20:15:47,068 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:15:47,759 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 20:15:49,496 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:15:52,065 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:15:53,811 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 20:15:53,812 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 20:15:57,902 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:15:59,497 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:16:02,903 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:16:04,498 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:16:07,904 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:16:08,811 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 20:16:08,812 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 20:16:12,947 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:16:14,508 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:16:16,226 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 20:16:16,228 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 20:16:16,228 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 20:16:16,229 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 20:16:16,789 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 20:16:17,069 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 20:16:17,790 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 20:16:18,071 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:16:19,630 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:16:23,072 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:16:23,811 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 20:16:23,811 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 20:16:28,969 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:16:29,631 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:16:33,969 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:16:34,632 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:16:38,811 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 20:16:38,812 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 20:16:39,961 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:16:43,898 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:16:44,964 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:16:47,071 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 20:16:48,696 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 20:16:48,698 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 20:16:48,699 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 20:16:48,700 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 20:16:48,817 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 20:16:49,131 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:16:50,701 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:16:51,819 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 20:16:53,811 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 20:16:53,812 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 20:16:55,904 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:16:59,132 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:17:00,905 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:17:04,133 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:17:05,905 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:17:08,811 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 20:17:08,812 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 20:17:10,962 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:17:14,134 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:17:15,963 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:17:17,075 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 20:17:19,142 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:17:21,083 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:17:23,815 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 20:17:23,816 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 20:17:24,676 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 20:17:24,678 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 20:17:24,678 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 20:17:24,680 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 20:17:24,845 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 20:17:25,846 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 20:17:26,681 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:17:28,822 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:17:31,682 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:17:33,822 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:17:36,682 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:17:38,815 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 20:17:38,816 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 20:17:41,914 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:17:43,824 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:17:46,914 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:17:47,080 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 20:17:48,824 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:17:52,085 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:17:53,820 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 20:17:53,823 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 20:17:56,212 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 20:17:56,214 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 20:17:56,215 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 20:17:56,216 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 20:17:56,872 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 20:17:57,217 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:17:57,873 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 20:17:58,832 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:18:02,217 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:18:03,833 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:18:07,218 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:18:08,817 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 20:18:08,818 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 20:18:12,902 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:18:14,044 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:18:17,083 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 20:18:18,085 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:18:19,045 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:18:23,092 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:18:23,830 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 20:18:23,833 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 20:18:26,116 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 20:18:26,117 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 20:18:26,118 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 20:18:26,120 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 20:18:26,897 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 20:18:27,898 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 20:18:28,121 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:18:29,056 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:18:33,121 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:18:34,057 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:18:38,122 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:18:38,892 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 20:18:38,893 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 20:18:44,052 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:18:44,058 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:18:47,086 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 20:18:49,059 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:18:49,088 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:18:53,899 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 20:18:53,900 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 20:18:55,063 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:18:56,658 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 20:18:56,660 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 20:18:56,660 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 20:18:56,662 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 20:18:56,925 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 20:18:57,926 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 20:18:59,071 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:19:00,663 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:19:04,072 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:19:05,664 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:19:08,895 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 20:19:08,895 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 20:19:11,001 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:19:14,139 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:19:16,002 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:19:17,090 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 20:19:19,139 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:19:21,093 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:19:23,905 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 20:19:23,914 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 20:19:27,034 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:19:27,732 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 20:19:27,734 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 20:19:27,734 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 20:19:27,736 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 20:19:27,952 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 20:19:29,160 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:19:29,954 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 20:19:32,737 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:19:34,161 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:19:37,738 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:19:38,899 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 20:19:38,900 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 20:19:43,033 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:19:44,328 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:19:47,094 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 20:19:48,096 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:19:49,328 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:19:49,329 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:19:53,097 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:19:53,899 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 20:19:53,900 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 20:19:59,020 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:19:59,340 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:20:01,980 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 20:20:01,981 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 20:20:01,982 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 20:20:01,984 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 20:20:01,989 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 20:20:03,991 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 20:20:04,380 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:20:04,380 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:20:04,985 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:20:08,899 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 20:20:08,900 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 20:20:10,971 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:20:14,381 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:20:14,382 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:20:15,972 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:20:17,099 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 20:20:19,382 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:20:19,382 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:20:21,102 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:20:23,899 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 20:20:23,900 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 20:20:26,968 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:20:29,389 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:20:31,974 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:20:32,706 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 20:20:32,707 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 20:20:32,708 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 20:20:32,709 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 20:20:33,023 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 20:20:34,024 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 20:20:34,393 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:20:37,710 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:20:38,899 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 20:20:38,900 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 20:20:43,066 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:20:44,395 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:20:47,102 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 20:20:48,105 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:20:49,396 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:20:49,396 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:20:53,105 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:20:53,900 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 20:20:53,900 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 20:20:59,069 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:20:59,405 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:21:03,123 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 20:21:03,125 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 20:21:03,126 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 20:21:03,128 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 20:21:04,062 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 20:21:04,128 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:21:04,531 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:21:04,531 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:21:06,064 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 20:21:08,900 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 20:21:08,900 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 20:21:09,997 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:21:14,532 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:21:14,997 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:21:17,106 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 20:21:19,533 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:21:20,109 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:21:23,900 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 20:21:23,900 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 20:21:26,060 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:21:29,538 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:21:31,069 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:21:33,933 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 20:21:33,935 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 20:21:33,935 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 20:21:33,936 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 20:21:34,085 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 20:21:34,548 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:21:36,086 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 20:21:36,937 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:21:38,900 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 20:21:38,900 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 20:21:42,006 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:21:44,550 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:21:47,007 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:21:47,110 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 20:21:49,550 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:21:49,551 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:21:52,113 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:21:53,908 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 20:21:53,916 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 20:21:58,055 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:21:59,561 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:21:59,571 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 20:21:59,573 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 20:21:59,573 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 20:21:59,574 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 20:22:00,109 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 20:22:02,110 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 20:22:03,575 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:22:04,562 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:22:08,576 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:22:08,902 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 20:22:08,903 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 20:22:14,017 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:22:14,563 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:22:17,115 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 20:22:19,117 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:22:19,564 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:22:23,902 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 20:22:23,903 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 20:22:25,054 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:22:29,577 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:22:30,053 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:22:31,015 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 20:22:31,016 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 20:22:31,016 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 20:22:31,017 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 20:22:31,148 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 20:22:32,149 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 20:22:34,575 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:22:34,576 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:22:36,019 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:22:38,902 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 20:22:38,903 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 20:22:42,014 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:22:44,577 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:22:44,577 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:22:47,015 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:22:47,118 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 20:22:49,578 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:22:49,578 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:22:52,121 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:22:53,902 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 20:22:53,903 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 20:22:57,974 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:22:59,584 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:23:01,263 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 20:23:01,265 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 20:23:01,265 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 20:23:01,266 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 20:23:02,178 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 20:23:02,178 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 20:23:03,267 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:23:04,672 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:23:04,672 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:23:08,268 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:23:08,902 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 20:23:08,903 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 20:23:14,010 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:23:14,673 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:23:14,674 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:23:17,122 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 20:23:19,125 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:23:19,674 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:23:19,675 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:23:23,903 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 20:23:23,903 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 20:23:24,994 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:23:29,682 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:23:29,999 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:23:32,135 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 20:23:32,136 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 20:23:32,137 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 20:23:32,138 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 20:23:32,210 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 20:23:34,212 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 20:23:34,682 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:23:34,682 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:23:35,139 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:23:38,903 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 20:23:38,903 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 20:23:41,005 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:23:44,683 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:23:44,683 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:23:46,006 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:23:47,125 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 20:23:49,685 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:23:49,685 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:23:51,128 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:23:53,903 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 20:23:53,904 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 20:23:56,970 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:23:59,693 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:24:01,976 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:24:03,818 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 20:24:03,819 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 20:24:03,820 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 20:24:03,821 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 20:24:04,236 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 20:24:04,691 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:24:06,238 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 20:24:07,823 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:24:08,903 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 20:24:08,903 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 20:24:13,032 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:24:14,692 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:24:14,693 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:24:17,129 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 20:24:18,132 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:24:19,693 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:24:23,132 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:24:23,903 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 20:24:23,903 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 20:24:28,995 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:24:29,703 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:24:34,001 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:24:34,700 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:24:34,855 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 20:24:34,857 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 20:24:34,858 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 20:24:34,859 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 20:24:35,264 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 20:24:36,265 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 20:24:38,903 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 20:24:38,904 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 20:24:39,001 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:24:44,002 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:24:44,701 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:24:47,133 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 20:24:49,136 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:24:49,710 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:24:53,903 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 20:24:53,904 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 20:24:55,060 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:24:59,719 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:25:00,063 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:25:04,719 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:25:04,826 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 20:25:04,828 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 20:25:04,828 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 20:25:04,830 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 20:25:05,317 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 20:25:05,831 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:25:06,318 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 20:25:08,903 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 20:25:08,904 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 20:25:10,973 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:25:14,721 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:25:15,974 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:25:17,137 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 20:25:19,722 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:25:21,140 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:25:23,903 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 20:25:23,904 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 20:25:27,025 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:25:29,724 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:25:32,029 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:25:34,733 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:25:35,913 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 20:25:35,915 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 20:25:35,916 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 20:25:35,917 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 20:25:36,341 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 20:25:37,919 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:25:38,343 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 20:25:38,904 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 20:25:38,904 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 20:25:43,059 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:25:44,731 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:25:47,142 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 20:25:48,144 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:25:49,760 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:25:53,145 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:25:53,904 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 20:25:53,904 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 20:25:59,058 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:25:59,761 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:26:04,069 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:26:04,772 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:26:06,658 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 20:26:06,660 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 20:26:06,660 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 20:26:06,662 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 20:26:07,369 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 20:26:08,370 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 20:26:08,904 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 20:26:08,905 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 20:26:09,971 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:26:14,805 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:26:14,805 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:26:14,971 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:26:17,144 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 20:26:19,806 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:26:19,806 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:26:20,146 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:26:23,904 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 20:26:23,905 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 20:26:26,001 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:26:29,816 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:26:31,005 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:26:34,812 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:26:35,044 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 20:26:35,046 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 20:26:35,046 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 20:26:35,047 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 20:26:35,398 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 20:26:36,048 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:26:36,399 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 20:26:38,904 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 20:26:38,905 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 20:26:42,021 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:26:44,814 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:26:47,021 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:26:47,148 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 20:26:49,905 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:26:49,906 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:26:52,150 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:26:53,904 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 20:26:53,905 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 20:26:57,964 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:26:58,907 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:26:58,907 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:27:02,978 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:27:03,918 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:27:05,684 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 20:27:05,686 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 20:27:05,686 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 20:27:05,687 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 20:27:06,428 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 20:27:06,428 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 20:27:08,688 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:27:08,904 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 20:27:08,905 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 20:27:13,985 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:27:14,084 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:27:14,085 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:27:17,152 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 20:27:19,086 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:27:19,086 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:27:19,155 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:27:23,905 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 20:27:23,905 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 20:27:25,001 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:27:29,087 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:27:29,088 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:27:30,007 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:27:34,097 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:27:35,012 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:27:35,262 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 20:27:35,264 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 20:27:35,264 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 20:27:35,266 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 20:27:35,458 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 20:27:36,460 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 20:27:38,905 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 20:27:38,905 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 20:27:40,022 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:27:44,094 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:27:45,022 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:27:47,156 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 20:27:49,094 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:27:50,159 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:27:53,904 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 20:27:53,905 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 20:27:56,006 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:27:59,096 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:28:01,007 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:28:04,104 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:28:06,013 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:28:07,030 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 20:28:07,031 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 20:28:07,031 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 20:28:07,036 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 20:28:07,492 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 20:28:08,493 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 20:28:08,905 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 20:28:08,905 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 20:28:11,065 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:28:14,431 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:28:14,432 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:28:16,065 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:28:17,160 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 20:28:19,432 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:28:19,433 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:28:21,163 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:28:23,905 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 20:28:23,905 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 20:28:27,028 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:28:29,434 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:28:29,434 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:28:32,028 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:28:34,441 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:28:37,036 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:28:38,893 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 20:28:38,895 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 20:28:38,896 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 20:28:38,897 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 20:28:39,296 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 20:28:39,296 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 20:28:39,522 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 20:28:40,523 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 20:28:42,460 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:28:44,442 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:28:47,165 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 20:28:48,167 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:28:49,443 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:28:53,168 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:28:54,041 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 20:28:54,042 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 20:28:59,153 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:28:59,444 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:29:04,160 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:29:04,452 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:29:08,433 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 20:29:08,434 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 20:29:08,434 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 20:29:08,436 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 20:29:08,549 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 20:29:09,041 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 20:29:09,042 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 20:29:10,123 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:29:10,551 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 20:29:14,584 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:29:14,585 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:29:15,123 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:29:17,168 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 20:29:19,585 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:29:19,586 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:29:20,170 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:29:24,041 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 20:29:24,042 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 20:29:25,252 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:29:29,587 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:29:29,587 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:29:30,252 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:29:34,588 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:29:34,588 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:29:35,257 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:29:39,045 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 20:29:39,046 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 20:29:39,788 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 20:29:39,789 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 20:29:39,790 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 20:29:39,791 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 20:29:40,581 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 20:29:40,582 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 20:29:40,792 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:29:44,592 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:29:45,792 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:29:47,173 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 20:29:49,593 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:29:51,175 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:29:54,044 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 20:29:54,045 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 20:29:57,160 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:29:59,594 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:30:02,160 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:30:04,595 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:30:07,170 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:30:09,044 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 20:30:09,044 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 20:30:11,591 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 20:30:11,591 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 20:30:11,592 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 20:30:11,593 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 20:30:11,614 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 20:30:12,594 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:30:12,616 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 20:30:14,738 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:30:17,177 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 20:30:18,179 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:30:19,738 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:30:19,739 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:30:23,180 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:30:24,044 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 20:30:24,045 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 20:30:29,114 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:30:29,740 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:30:29,741 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:30:34,114 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:30:34,741 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:30:34,741 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:30:39,047 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 20:30:39,048 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 20:30:39,199 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:30:43,686 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 20:30:43,687 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 20:30:43,688 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 20:30:43,689 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 20:30:44,650 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 20:30:44,650 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 20:30:44,690 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:30:44,832 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:30:44,832 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:30:47,181 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 20:30:49,833 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:30:49,834 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:30:50,183 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:30:54,045 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 20:30:54,045 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 20:30:56,127 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:30:59,835 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:30:59,835 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:31:01,128 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:31:04,836 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:31:04,836 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:31:06,128 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:31:09,045 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 20:31:09,045 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 20:31:12,124 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:31:14,841 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:31:17,133 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:31:17,186 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 20:31:17,336 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 20:31:17,338 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 20:31:17,338 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 20:31:17,340 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 20:31:17,680 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 20:31:18,682 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 20:31:19,841 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:31:22,341 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:31:24,045 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 20:31:24,045 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 20:31:28,211 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:31:29,842 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:31:33,212 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:31:34,843 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:31:38,212 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:31:39,045 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 20:31:39,045 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 20:31:44,144 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:31:44,853 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:31:47,191 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 20:31:48,396 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 20:31:48,398 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 20:31:48,399 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 20:31:48,400 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 20:31:48,707 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 20:31:48,707 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 20:31:49,400 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:31:49,852 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:31:54,045 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 20:31:54,046 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 20:31:55,114 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:31:59,853 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:32:00,114 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:32:04,854 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:32:05,115 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:32:09,045 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 20:32:09,046 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 20:32:11,113 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:32:14,862 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:32:16,119 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:32:17,195 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 20:32:18,938 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 20:32:18,939 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 20:32:18,939 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 20:32:18,941 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 20:32:19,733 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 20:32:19,867 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:32:20,735 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 20:32:21,942 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:32:24,045 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 20:32:24,046 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 20:32:27,114 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:32:29,868 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:32:32,115 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:32:34,972 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:32:37,116 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:32:39,045 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 20:32:39,046 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 20:32:42,180 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:32:44,980 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:32:47,187 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:32:47,197 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 20:32:47,997 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 20:32:47,999 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 20:32:47,999 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 20:32:48,001 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 20:32:48,769 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 20:32:48,770 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 20:32:49,981 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:32:53,002 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:32:54,045 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 20:32:54,046 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 20:32:58,112 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:32:59,982 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:33:03,113 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:33:04,983 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:33:08,113 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:33:09,045 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 20:33:09,046 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 20:33:13,150 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:33:14,984 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:33:17,209 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 20:33:18,212 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:33:19,988 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:33:21,390 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 20:33:21,391 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 20:33:21,392 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 20:33:21,394 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 20:33:21,801 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 20:33:22,802 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 20:33:23,395 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:33:24,046 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 20:33:24,046 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 20:33:29,112 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:33:29,990 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:33:34,113 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:33:34,991 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:33:39,046 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 20:33:39,046 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 20:33:39,202 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:33:44,203 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:33:44,993 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:33:47,209 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 20:33:49,211 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:33:50,000 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:33:51,978 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 20:33:51,979 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 20:33:51,980 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 20:33:51,981 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 20:33:52,833 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 20:33:52,834 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 20:33:54,046 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 20:33:54,047 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 20:33:55,195 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:33:59,125 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:33:59,125 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:34:00,195 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:34:04,125 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:34:04,126 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:34:05,196 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:34:09,046 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 20:34:09,046 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 20:34:10,204 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:34:14,127 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:34:14,127 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:34:15,205 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:34:17,214 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 20:34:19,131 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:34:20,223 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:34:22,699 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 20:34:22,700 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 20:34:22,701 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 20:34:22,706 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 20:34:22,862 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 20:34:24,046 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 20:34:24,047 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 20:34:24,864 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 20:34:26,198 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:34:29,136 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:34:31,198 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:34:34,137 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:34:36,199 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:34:39,046 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 20:34:39,047 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 20:34:42,187 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:34:44,139 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:34:44,139 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:34:47,193 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:34:47,215 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 20:34:49,150 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:34:50,863 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 20:34:50,865 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 20:34:50,865 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 20:34:50,867 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 20:34:50,895 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 20:34:52,868 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:34:52,901 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 20:34:54,046 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 20:34:54,047 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 20:34:58,143 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:34:59,266 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:34:59,266 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:35:03,144 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:35:04,267 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:35:04,268 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:35:08,144 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:35:09,046 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 20:35:09,047 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 20:35:14,151 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:35:14,274 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:35:17,220 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 20:35:19,222 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:35:19,276 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:35:20,022 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 20:35:20,024 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 20:35:20,024 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 20:35:20,026 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 20:35:20,929 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 20:35:21,930 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 20:35:24,047 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 20:35:24,047 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 20:35:25,221 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:35:29,419 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:35:30,221 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:35:34,420 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:35:35,222 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:35:39,047 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 20:35:39,047 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 20:35:41,182 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:35:44,421 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:35:44,422 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:35:46,187 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:35:47,224 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 20:35:49,426 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:35:49,426 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:35:50,986 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 20:35:50,987 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 20:35:50,988 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 20:35:50,989 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 20:35:51,958 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 20:35:51,990 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:35:53,960 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 20:35:54,047 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 20:35:54,047 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 20:35:57,138 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:35:59,431 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:36:02,139 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:36:04,431 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:36:04,431 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:36:07,140 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:36:09,047 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 20:36:09,048 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 20:36:12,211 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:36:14,433 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:36:14,433 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:36:17,221 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:36:17,225 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 20:36:19,438 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:36:22,227 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:36:22,796 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 20:36:22,798 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 20:36:22,798 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 20:36:22,799 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 20:36:22,994 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 20:36:23,995 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 20:36:24,047 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 20:36:24,048 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 20:36:28,186 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:36:29,441 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:36:33,187 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:36:34,485 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:36:34,486 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:36:38,187 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:36:39,047 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 20:36:39,048 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 20:36:44,115 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:36:44,487 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:36:47,233 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 20:36:49,237 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:36:49,501 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:36:52,381 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 20:36:52,383 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 20:36:52,384 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 20:36:52,385 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 20:36:53,019 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 20:36:54,020 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 20:36:54,047 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 20:36:54,048 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 20:36:55,150 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:36:59,537 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:36:59,537 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:37:00,150 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:37:04,538 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:37:04,538 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:37:05,151 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:37:09,047 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 20:37:09,048 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 20:37:10,167 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:37:14,539 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:37:15,167 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:37:17,236 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 20:37:19,543 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:37:20,245 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:37:24,047 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 20:37:24,048 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 20:37:24,077 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 20:37:24,144 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 20:37:24,144 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 20:37:24,146 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 20:37:25,053 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 20:37:26,054 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 20:37:26,147 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:37:29,547 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:37:31,148 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:37:34,547 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:37:36,148 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:37:39,048 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 20:37:39,048 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 20:37:41,191 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:37:44,549 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:37:46,192 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:37:47,238 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 20:37:49,555 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:37:51,245 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:37:54,048 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 20:37:54,048 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 20:37:54,314 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 20:37:54,316 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 20:37:54,316 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 20:37:54,318 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 20:37:55,074 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 20:37:56,075 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 20:37:56,319 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:37:59,712 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:38:01,319 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:38:04,713 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:38:06,320 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:38:09,048 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 20:38:09,049 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 20:38:12,150 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:38:14,715 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:38:14,715 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:38:17,156 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:38:17,246 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 20:38:19,726 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:38:22,257 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:38:22,988 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 20:38:22,989 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 20:38:22,989 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 20:38:22,991 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 20:38:23,119 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 20:38:24,054 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 20:38:24,055 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 20:38:26,121 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 20:38:28,197 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:38:29,722 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:38:33,198 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:38:34,723 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:38:38,198 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:38:39,054 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 20:38:39,054 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 20:38:43,227 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:38:44,730 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:38:47,247 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 20:38:48,248 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:38:49,728 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:38:49,758 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 20:38:49,760 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 20:38:49,760 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 20:38:49,765 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 20:38:50,149 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 20:38:52,150 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 20:38:53,767 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:38:54,054 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 20:38:54,054 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 20:38:59,221 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:38:59,729 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:39:04,221 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:39:04,730 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:39:09,054 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 20:39:09,055 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 20:39:10,161 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:39:14,734 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:39:15,167 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:39:17,255 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 20:39:19,738 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:39:20,164 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 20:39:20,166 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 20:39:20,166 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 20:39:20,170 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 20:39:20,171 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:39:20,175 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 20:39:22,177 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 20:39:24,054 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 20:39:24,055 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 20:39:26,144 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:39:29,739 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:39:31,145 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:39:34,740 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:39:36,146 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:39:39,054 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 20:39:39,055 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 20:39:41,164 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:39:44,741 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:39:46,165 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:39:47,258 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 20:39:49,742 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:39:51,266 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:39:54,062 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 20:39:54,062 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 20:39:56,028 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 20:39:56,029 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 20:39:56,029 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 20:39:56,031 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 20:39:56,209 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 20:39:57,032 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:39:58,211 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 20:39:59,747 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:40:02,032 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:40:04,748 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:40:07,033 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:40:09,059 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 20:40:09,060 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 20:40:12,138 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:40:14,749 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:40:17,140 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:40:17,262 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 20:40:19,755 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:40:22,272 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:40:24,059 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 20:40:24,060 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 20:40:24,113 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 20:40:24,222 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 20:40:24,223 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 20:40:24,224 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 20:40:24,237 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 20:40:26,239 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 20:40:28,226 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:40:29,756 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:40:33,226 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:40:34,756 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:40:38,227 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:40:39,059 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 20:40:39,060 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 20:40:44,226 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:40:44,814 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:40:47,266 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 20:40:49,269 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:40:49,826 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:40:54,062 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 20:40:54,062 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 20:40:55,205 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:40:55,803 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 20:40:55,805 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 20:40:55,805 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 20:40:55,807 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 20:40:56,259 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 20:40:58,260 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 20:40:59,948 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:41:00,808 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:41:04,949 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:41:05,809 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:41:09,062 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 20:41:09,063 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 20:41:11,197 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:41:14,950 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:41:16,197 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:41:17,270 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 20:41:19,951 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:41:21,276 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:41:24,066 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 20:41:24,067 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 20:41:26,096 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 20:41:26,097 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 20:41:26,098 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 20:41:26,099 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 20:41:26,293 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 20:41:27,100 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:41:28,295 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 20:41:29,956 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:41:29,957 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:41:32,101 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:41:34,958 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:41:34,958 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:41:37,101 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:41:39,077 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 20:41:39,077 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 20:41:42,243 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:41:45,076 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:41:45,076 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:41:47,243 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:41:47,274 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 20:41:50,077 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:41:50,077 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:41:52,278 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:41:54,082 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 20:41:54,083 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 20:41:58,125 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 20:41:58,127 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 20:41:58,127 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 20:41:58,127 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:41:58,128 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 20:41:58,324 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 20:41:59,086 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:41:59,087 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:42:00,326 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 20:42:03,129 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:42:04,087 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:42:04,087 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:42:08,130 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:42:09,079 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 20:42:09,080 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 20:42:13,192 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:42:14,089 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:42:14,089 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:42:17,279 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 20:42:18,281 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:42:19,090 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:42:19,090 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:42:23,287 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:42:24,087 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 20:42:24,088 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 20:42:29,054 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 20:42:29,056 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 20:42:29,056 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 20:42:29,056 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:42:29,057 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 20:42:29,354 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 20:42:29,451 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:42:29,452 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:42:30,355 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 20:42:34,058 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:42:34,453 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:42:34,453 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:42:39,059 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:42:39,084 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 20:42:39,085 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 20:42:44,234 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:42:44,455 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:42:44,455 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:42:47,282 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 20:42:49,284 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:42:49,456 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:42:49,456 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:42:54,089 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 20:42:54,093 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 20:42:55,237 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:42:55,971 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 20:42:55,972 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 20:42:55,972 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 20:42:55,973 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 20:42:56,376 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 20:42:58,379 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 20:42:59,468 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:42:59,468 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:43:00,974 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:43:04,469 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:43:04,469 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:43:05,975 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:43:09,088 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 20:43:09,089 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 20:43:11,231 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:43:14,470 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:43:14,471 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:43:16,231 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:43:17,286 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 20:43:19,478 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:43:21,298 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:43:24,089 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 20:43:24,089 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 20:43:24,520 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 20:43:24,522 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 20:43:24,523 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 20:43:24,524 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 20:43:25,411 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 20:43:26,412 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 20:43:26,525 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:43:29,478 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:43:31,526 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:43:34,479 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:43:36,527 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:43:39,093 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 20:43:39,094 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 20:43:42,194 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:43:44,480 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:43:44,481 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:43:47,194 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:43:47,290 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 20:43:49,486 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:43:52,301 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:43:54,098 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 20:43:54,098 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 20:43:55,413 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 20:43:55,415 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 20:43:55,415 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 20:43:55,417 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 20:43:55,438 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 20:43:56,439 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 20:43:57,418 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:43:59,813 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:43:59,814 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:44:02,419 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:44:04,814 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:44:04,815 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:44:07,419 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:44:09,098 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 20:44:09,099 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 20:44:13,251 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:44:14,864 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:44:17,295 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 20:44:18,303 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:44:19,875 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:44:23,305 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:44:23,478 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 20:44:23,480 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 20:44:23,480 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 20:44:23,482 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 20:44:24,098 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 20:44:24,099 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 20:44:24,463 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 20:44:24,464 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 20:44:29,227 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:44:29,879 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:44:29,879 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:44:34,227 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:44:34,879 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:44:34,880 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:44:39,098 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 20:44:39,099 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 20:44:39,259 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:44:44,260 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:44:44,881 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:44:44,881 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:44:47,300 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 20:44:49,308 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:44:49,885 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:44:54,099 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 20:44:54,099 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 20:44:54,237 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 20:44:54,238 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 20:44:54,239 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 20:44:54,241 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 20:44:54,492 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 20:44:55,241 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:44:56,493 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 20:44:59,888 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:45:00,242 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:45:04,888 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:45:05,243 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:45:09,098 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 20:45:09,099 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 20:45:10,255 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:45:14,890 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:45:15,255 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:45:17,303 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 20:45:19,898 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:45:19,898 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:45:20,306 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:45:24,099 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 20:45:24,099 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 20:45:26,257 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:45:29,445 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:45:29,534 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:45:30,513 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 20:45:31,796 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:45:36,797 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:45:38,546 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:45:38,848 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:45:40,461 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 20:45:40,462 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 20:45:42,630 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:45:47,307 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 20:45:48,290 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:45:48,309 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:45:48,460 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:45:53,310 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:45:58,311 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:45:58,472 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:45:59,622 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 20:45:59,622 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 20:46:03,724 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:46:08,725 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:46:09,958 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:46:10,482 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:46:13,725 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:46:17,311 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 20:46:19,314 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:46:21,364 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:46:21,426 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:46:21,791 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 20:46:21,792 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 20:46:24,975 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:46:29,976 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:46:33,183 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:46:33,313 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:46:34,977 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:46:38,614 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:46:38,614 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 20:46:38,615 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 20:46:40,682 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 20:46:40,950 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:46:42,683 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 20:46:43,615 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:46:45,951 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:46:47,315 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 20:46:48,185 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 20:46:48,185 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 20:46:51,298 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:46:53,616 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:46:56,299 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:46:58,618 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:46:58,618 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:47:01,301 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:47:03,188 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 20:47:03,188 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 20:47:06,302 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:47:08,619 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:47:08,620 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:47:11,303 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:47:13,620 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:47:13,621 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:47:16,304 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:47:17,320 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 20:47:18,185 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 20:47:18,185 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 20:47:21,325 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:47:23,622 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:47:26,326 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:47:31,327 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:47:36,327 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:47:41,328 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:47:44,478 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:47:44,483 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 20:47:44,483 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 20:47:46,587 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:47:47,323 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 20:47:49,483 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:47:49,483 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:47:52,326 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:47:54,484 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:47:54,484 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:47:57,327 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:47:59,483 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 20:47:59,483 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 20:48:02,607 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:48:04,485 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:48:07,607 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:48:09,486 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:48:09,487 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:48:12,608 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:48:14,483 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 20:48:14,484 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 20:48:17,327 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 20:48:18,329 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:48:19,488 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:48:19,488 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:48:23,330 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:48:24,489 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:48:24,489 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:48:28,331 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:48:29,483 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 20:48:29,483 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 20:48:33,578 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:48:34,491 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:48:38,578 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:48:39,491 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:48:39,492 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:48:43,579 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:48:44,483 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 20:48:44,484 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 20:48:47,330 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 20:48:49,332 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:48:49,494 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:48:49,494 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:48:54,333 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:48:54,495 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:48:54,495 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:48:59,333 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:48:59,483 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 20:48:59,484 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 20:49:04,496 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:49:04,560 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:49:09,497 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:49:09,561 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:49:14,483 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 20:49:14,484 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 20:49:14,649 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:49:17,334 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 20:49:19,498 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:49:20,337 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:49:24,499 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:49:25,342 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:49:29,483 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 20:49:29,484 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 20:49:30,581 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:49:34,501 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:49:35,582 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:49:39,502 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:49:40,583 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:49:44,484 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 20:49:44,484 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 20:49:46,564 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:49:47,338 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 20:49:52,341 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:49:57,342 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:50:02,342 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:50:07,347 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:50:12,347 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:50:17,342 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 20:50:18,345 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:50:19,685 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:50:19,690 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:50:19,690 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 20:50:19,690 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 20:50:22,805 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 20:50:23,746 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:50:24,691 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:50:24,691 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:50:26,808 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 20:50:29,084 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:50:29,692 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:50:29,692 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:50:34,085 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:50:34,687 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 20:50:34,687 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 20:50:39,694 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:50:39,694 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:50:39,801 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:50:44,695 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:50:44,695 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:50:44,801 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:50:47,348 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 20:50:49,695 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 20:50:49,698 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 20:50:50,801 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:50:52,140 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 20:50:52,142 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 20:50:52,142 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 20:50:52,143 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 20:50:52,833 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 20:50:54,704 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:50:54,705 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:50:54,836 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 20:50:56,144 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:50:59,705 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:50:59,706 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:51:01,144 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:51:04,689 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 20:51:04,690 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 20:51:06,811 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:51:09,707 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:51:11,812 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:51:14,709 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:51:14,709 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:51:16,812 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:51:17,352 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 20:51:19,689 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 20:51:19,690 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 20:51:22,799 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:51:24,719 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:51:24,719 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:51:25,997 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 20:51:25,998 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 20:51:25,998 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 20:51:25,999 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 20:51:26,869 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 20:51:26,870 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 20:51:28,000 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:51:29,720 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:51:29,720 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:51:33,001 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:51:34,689 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 20:51:34,690 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 20:51:38,830 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:51:39,722 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:51:39,722 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:51:43,830 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:51:44,723 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:51:44,723 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:51:47,356 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 20:51:49,358 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:51:49,689 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 20:51:49,690 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 20:51:54,736 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:51:54,871 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:51:56,166 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 20:51:56,168 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 20:51:56,169 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 20:51:56,170 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 20:51:56,906 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 20:51:58,908 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 20:51:59,731 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:52:00,172 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:52:04,689 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 20:52:04,690 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 20:52:05,775 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:52:09,733 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:52:10,776 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:52:14,733 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:52:15,776 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:52:17,360 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 20:52:19,690 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 20:52:19,690 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 20:52:20,840 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:52:24,742 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:52:25,855 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:52:27,735 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 20:52:27,737 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 20:52:27,738 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 20:52:27,739 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 20:52:27,934 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 20:52:28,935 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 20:52:29,881 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:52:29,881 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:52:31,740 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:52:34,690 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 20:52:34,690 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 20:52:36,818 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:52:39,883 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:52:41,818 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:52:44,884 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:52:44,884 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:52:46,819 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:52:47,364 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 20:52:49,690 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 20:52:49,690 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 20:52:51,852 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:52:54,888 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:52:56,859 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:52:59,842 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 20:52:59,843 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 20:52:59,844 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 20:52:59,845 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 20:52:59,958 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 20:52:59,988 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:53:00,284 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:53:00,959 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 20:53:02,846 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:53:04,690 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 20:53:04,691 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 20:53:08,802 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:53:10,285 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:53:10,285 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:53:13,802 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:53:15,286 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:53:15,286 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:53:17,369 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 20:53:19,371 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:53:19,690 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 20:53:19,691 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 20:53:24,856 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:53:25,288 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:53:25,288 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:53:29,863 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:53:30,296 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:53:30,297 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:53:31,447 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 20:53:31,448 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 20:53:31,449 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 20:53:31,450 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 20:53:31,984 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 20:53:32,985 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 20:53:34,690 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 20:53:34,691 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 20:53:35,842 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:53:40,297 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:53:40,298 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:53:40,843 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:53:45,368 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:53:45,368 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:53:45,843 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:53:47,372 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 20:53:49,691 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 20:53:49,691 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 20:53:51,782 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:53:55,377 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:53:56,787 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:53:59,596 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 20:53:59,597 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 20:53:59,597 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 20:53:59,599 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 20:54:00,011 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 20:54:00,383 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:54:00,384 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:54:01,012 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 20:54:02,600 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:54:04,691 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 20:54:04,691 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 20:54:07,780 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:54:10,385 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:54:12,781 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:54:15,386 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:54:15,386 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:54:17,376 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 20:54:18,378 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:54:19,691 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 20:54:19,691 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 20:54:23,800 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:54:25,392 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:54:28,810 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:54:30,394 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:54:30,469 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 20:54:30,471 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 20:54:30,471 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 20:54:30,472 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 20:54:31,041 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 20:54:33,042 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 20:54:34,473 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:54:34,691 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 20:54:34,691 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 20:54:39,816 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:54:40,395 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:54:44,817 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:54:45,396 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:54:47,380 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 20:54:49,691 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 20:54:49,691 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 20:54:49,848 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:54:54,849 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:54:55,398 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:54:59,856 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:55:00,402 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:55:01,429 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 20:55:01,430 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 20:55:01,431 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 20:55:01,432 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 20:55:02,066 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 20:55:03,067 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 20:55:04,691 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 20:55:04,692 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 20:55:05,812 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:55:10,404 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:55:10,812 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:55:15,405 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:55:15,813 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:55:17,382 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 20:55:19,691 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 20:55:19,692 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 20:55:20,820 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:55:25,407 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:55:25,825 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:55:30,415 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:55:30,827 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:55:31,575 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 20:55:31,577 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 20:55:31,577 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 20:55:31,579 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 20:55:32,088 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 20:55:33,088 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 20:55:34,691 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 20:55:34,692 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 20:55:36,820 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:55:40,416 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:55:41,821 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:55:45,417 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:55:46,821 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:55:47,387 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 20:55:49,691 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 20:55:49,692 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 20:55:52,803 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:55:55,419 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:55:55,419 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:55:57,806 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:56:00,430 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:56:01,476 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 20:56:01,477 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 20:56:01,478 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 20:56:01,479 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 20:56:02,115 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 20:56:03,116 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 20:56:03,480 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:56:04,691 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 20:56:04,692 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 20:56:08,773 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:56:10,429 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:56:13,774 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:56:15,430 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:56:17,390 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 20:56:19,393 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:56:19,691 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 20:56:19,692 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 20:56:24,859 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:56:25,431 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:56:29,867 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:56:30,443 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:56:32,158 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 20:56:32,159 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 20:56:32,160 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 20:56:32,161 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 20:56:33,141 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 20:56:33,141 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 20:56:34,692 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 20:56:34,692 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 20:56:35,852 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:56:40,565 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:56:40,852 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:56:45,566 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:56:45,566 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:56:45,853 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:56:47,394 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 20:56:49,692 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 20:56:49,692 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 20:56:51,808 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:56:55,579 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:56:55,579 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:56:56,817 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:56:59,335 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 20:56:59,336 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 20:56:59,337 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 20:56:59,338 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 20:57:00,162 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 20:57:00,744 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:57:00,744 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:57:01,162 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 20:57:02,339 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:57:04,692 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 20:57:04,693 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 20:57:07,861 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:57:09,745 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:57:12,861 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:57:14,746 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:57:17,398 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 20:57:18,401 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:57:19,692 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 20:57:19,693 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 20:57:23,772 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:57:24,751 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:57:28,779 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:57:29,592 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 20:57:29,593 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 20:57:29,594 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 20:57:29,595 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 20:57:29,994 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:57:29,994 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:57:30,186 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 20:57:31,187 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 20:57:34,596 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:57:34,693 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 20:57:34,693 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 20:57:39,821 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:57:39,996 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:57:39,996 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:57:44,822 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:57:44,997 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:57:47,402 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 20:57:49,692 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 20:57:49,693 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 20:57:50,758 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:57:54,998 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:57:55,765 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:58:00,003 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:58:00,738 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 20:58:00,739 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 20:58:00,739 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 20:58:00,741 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 20:58:01,214 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 20:58:01,741 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:58:03,215 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 20:58:04,693 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 20:58:04,693 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 20:58:06,805 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:58:10,137 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:58:11,806 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:58:15,138 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:58:15,139 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:58:16,806 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:58:17,407 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 20:58:19,693 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 20:58:19,693 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 20:58:22,784 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:58:25,181 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:58:27,785 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:58:30,189 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:58:32,795 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:58:34,693 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 20:58:34,694 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 20:58:34,854 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 20:58:34,856 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 20:58:34,857 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 20:58:34,859 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 20:58:35,244 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 20:58:37,246 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 20:58:37,860 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:58:40,263 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:58:40,263 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:58:42,860 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:58:45,264 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:58:45,264 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:58:47,411 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 20:58:48,413 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:58:49,693 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 20:58:49,694 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 20:58:53,768 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:58:55,265 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:58:55,265 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:58:58,769 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:59:00,269 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:59:03,778 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:59:04,694 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 20:59:04,694 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 20:59:05,088 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 20:59:05,090 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 20:59:05,090 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 20:59:05,092 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 20:59:05,271 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 20:59:07,273 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 20:59:09,093 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:59:10,496 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:59:14,094 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:59:15,497 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:59:17,413 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 20:59:19,415 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:59:19,694 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 20:59:19,695 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 20:59:24,869 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:59:25,635 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:59:25,635 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:59:29,869 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:59:30,636 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:59:30,637 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:59:34,701 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 20:59:34,702 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 20:59:35,826 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:59:36,593 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 20:59:36,595 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 20:59:36,595 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 20:59:36,596 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 20:59:37,306 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 20:59:37,306 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 20:59:39,740 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:59:41,598 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:59:44,740 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:59:46,598 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:59:47,417 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 20:59:49,698 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 20:59:49,698 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 20:59:51,799 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:59:54,741 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 20:59:56,800 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 20:59:59,742 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:00:01,801 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:00:04,707 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 21:00:04,713 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 21:00:06,811 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:00:07,850 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 21:00:07,852 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 21:00:07,852 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 21:00:07,853 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 21:00:08,343 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 21:00:09,344 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 21:00:09,746 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:00:11,854 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:00:14,746 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:00:16,855 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:00:17,421 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 21:00:19,703 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 21:00:19,704 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 21:00:21,861 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:00:24,748 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:00:26,861 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:00:29,749 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:00:31,862 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:00:34,717 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 21:00:34,725 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 21:00:37,846 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:00:38,844 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 21:00:38,846 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 21:00:38,846 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 21:00:38,848 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 21:00:39,372 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 21:00:39,761 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:00:41,374 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 21:00:42,849 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:00:44,761 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:00:47,425 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 21:00:48,427 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:00:49,711 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 21:00:49,711 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 21:00:53,880 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:00:54,763 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:00:54,763 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:00:58,880 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:00:59,764 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:01:03,889 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:01:04,717 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 21:01:04,723 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 21:01:08,889 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:01:09,050 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 21:01:09,051 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 21:01:09,051 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 21:01:09,052 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 21:01:09,397 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 21:01:09,769 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:01:11,398 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 21:01:14,053 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:01:14,770 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:01:17,430 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 21:01:19,432 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:01:19,714 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 21:01:19,714 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 21:01:24,771 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:01:24,882 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:01:29,772 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:01:29,773 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:01:29,882 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:01:34,723 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 21:01:34,730 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 21:01:35,882 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:01:39,779 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:01:40,279 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 21:01:40,280 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 21:01:40,280 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 21:01:40,282 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 21:01:40,423 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 21:01:41,283 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:01:41,424 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 21:01:44,780 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:01:46,283 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:01:47,434 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 21:01:49,718 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 21:01:49,719 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 21:01:51,833 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:01:54,911 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:01:56,833 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:01:59,911 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:02:01,834 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:02:04,721 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 21:02:04,723 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 21:02:06,842 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:02:08,418 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 21:02:08,419 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 21:02:08,419 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 21:02:08,421 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 21:02:08,448 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 21:02:09,449 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 21:02:09,917 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:02:10,077 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:02:12,422 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:02:15,077 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:02:17,423 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:02:17,437 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 21:02:19,721 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 21:02:19,722 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 21:02:22,889 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:02:25,148 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:02:25,149 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:02:27,890 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:02:30,149 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:02:30,149 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:02:32,890 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:02:34,731 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 21:02:34,735 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 21:02:38,881 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:02:38,938 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 21:02:38,939 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 21:02:38,940 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 21:02:38,942 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 21:02:39,480 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 21:02:39,480 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 21:02:40,339 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:02:43,943 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:02:45,340 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:02:45,340 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:02:47,442 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 21:02:49,444 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:02:49,726 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 21:02:49,727 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 21:02:54,857 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:02:55,341 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:02:55,342 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:02:59,858 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:03:00,342 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:03:00,343 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:03:04,733 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 21:03:04,737 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 21:03:05,824 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:03:08,247 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 21:03:08,247 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 21:03:08,248 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 21:03:08,249 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 21:03:08,515 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 21:03:09,516 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 21:03:10,648 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:03:10,649 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:03:11,250 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:03:15,649 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:03:16,251 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:03:17,446 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 21:03:19,730 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 21:03:19,731 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 21:03:21,863 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:03:24,824 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:03:26,864 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:03:29,825 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:03:31,865 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:03:34,742 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 21:03:34,746 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 21:03:37,856 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:03:38,551 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 21:03:38,552 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 21:03:38,552 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 21:03:38,553 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 21:03:39,544 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 21:03:39,545 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 21:03:39,947 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:03:39,947 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:03:43,554 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:03:44,948 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:03:44,948 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:03:47,451 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 21:03:49,453 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:03:49,736 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 21:03:49,736 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 21:03:54,859 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:03:54,950 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:03:54,950 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:03:59,859 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:03:59,951 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:03:59,951 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:04:04,741 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 21:04:04,741 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 21:04:05,854 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:04:07,987 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 21:04:07,989 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 21:04:07,989 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 21:04:07,990 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 21:04:08,577 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 21:04:09,960 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:04:10,578 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 21:04:10,992 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:04:14,960 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:04:15,992 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:04:17,454 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 21:04:19,741 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 21:04:19,742 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 21:04:21,873 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:04:24,962 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:04:26,874 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:04:29,963 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:04:31,885 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:04:34,746 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 21:04:34,749 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 21:04:37,237 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 21:04:37,238 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 21:04:37,239 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 21:04:37,239 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:04:37,240 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 21:04:37,612 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 21:04:38,613 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 21:04:39,976 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:04:42,242 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:04:44,977 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:04:47,242 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:04:47,458 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 21:04:49,742 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 21:04:49,742 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 21:04:52,857 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:04:54,979 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:04:57,858 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:04:59,980 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:05:02,858 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:05:04,750 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 21:05:04,751 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 21:05:08,823 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:05:09,316 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 21:05:09,317 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 21:05:09,317 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 21:05:09,318 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 21:05:09,643 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 21:05:09,990 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:05:10,644 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 21:05:14,319 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:05:14,991 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:05:17,462 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 21:05:19,464 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:05:19,747 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 21:05:19,747 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 21:05:24,848 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:05:24,993 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:05:29,849 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:05:29,994 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:05:34,747 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 21:05:34,748 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 21:05:35,822 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:05:40,009 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:05:40,824 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:05:42,893 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 21:05:42,894 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 21:05:42,894 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 21:05:42,896 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 21:05:43,677 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 21:05:44,678 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 21:05:45,038 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:05:45,039 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:05:45,897 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:05:47,466 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 21:05:49,747 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 21:05:49,748 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 21:05:51,882 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:05:55,040 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:05:55,041 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:05:56,883 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:06:00,041 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:06:00,042 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:06:01,883 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:06:04,748 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 21:06:04,748 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 21:06:07,839 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:06:10,051 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:06:12,848 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:06:14,005 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 21:06:14,007 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 21:06:14,007 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 21:06:14,009 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 21:06:14,713 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 21:06:15,153 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:06:16,714 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 21:06:17,470 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 21:06:18,472 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:06:19,747 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 21:06:19,748 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 21:06:23,906 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:06:25,155 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:06:28,906 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:06:30,156 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:06:33,907 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:06:34,748 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 21:06:34,748 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 21:06:39,833 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:06:40,168 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:06:43,639 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 21:06:43,641 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 21:06:43,641 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 21:06:43,642 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 21:06:43,735 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 21:06:44,736 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 21:06:45,168 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:06:45,643 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:06:47,475 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 21:06:49,748 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 21:06:49,749 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 21:06:50,871 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:06:55,169 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:06:55,872 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:07:00,170 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:07:00,872 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:07:04,748 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 21:07:04,749 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 21:07:06,863 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:07:10,181 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:07:11,869 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:07:13,192 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 21:07:13,194 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 21:07:13,194 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 21:07:13,195 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 21:07:13,767 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 21:07:14,768 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 21:07:15,179 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:07:17,196 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:07:17,478 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 21:07:19,748 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 21:07:19,748 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 21:07:22,910 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:07:25,224 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:07:25,309 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:07:27,911 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:07:30,310 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:07:30,310 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:07:32,912 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:07:34,748 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 21:07:34,749 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 21:07:38,838 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:07:40,325 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:07:42,162 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 21:07:42,164 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 21:07:42,164 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 21:07:42,166 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 21:07:42,806 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 21:07:44,167 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:07:44,808 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 21:07:45,893 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:07:47,482 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 21:07:49,484 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:07:49,748 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 21:07:49,749 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 21:07:54,894 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:07:54,895 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:07:54,902 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:07:59,895 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:07:59,903 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:08:04,748 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 21:08:04,749 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 21:08:05,849 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:08:09,901 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:08:09,901 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:08:10,852 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:08:11,132 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 21:08:11,133 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 21:08:11,134 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 21:08:11,135 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 21:08:11,845 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 21:08:12,846 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 21:08:15,192 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:08:16,136 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:08:17,486 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 21:08:19,748 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 21:08:19,749 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 21:08:21,910 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:08:25,194 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:08:25,194 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:08:26,910 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:08:30,195 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:08:30,195 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:08:31,911 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:08:34,749 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 21:08:34,749 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 21:08:37,854 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:08:40,199 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:08:42,861 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:08:44,949 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 21:08:44,950 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 21:08:44,951 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 21:08:44,952 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 21:08:45,355 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:08:45,355 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:08:45,877 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 21:08:46,878 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 21:08:47,490 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 21:08:48,492 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:08:49,749 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 21:08:49,749 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 21:08:53,842 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:08:55,356 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:08:58,843 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:09:00,357 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:09:00,357 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:09:03,843 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:09:04,749 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 21:09:04,749 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 21:09:09,850 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:09:10,364 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:09:14,851 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:09:15,209 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 21:09:15,210 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 21:09:15,211 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 21:09:15,212 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 21:09:15,655 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:09:15,655 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:09:15,906 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 21:09:16,907 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 21:09:17,495 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 21:09:19,749 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 21:09:19,750 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 21:09:20,847 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:09:25,656 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:09:25,848 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:09:30,666 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:09:30,667 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:09:30,848 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:09:34,749 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 21:09:34,750 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 21:09:35,851 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:09:40,668 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:09:40,668 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:09:40,852 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:09:45,675 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:09:45,676 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:09:45,855 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:09:47,498 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 21:09:49,155 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 21:09:49,156 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 21:09:49,156 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 21:09:49,157 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 21:09:49,749 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 21:09:49,750 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 21:09:49,939 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 21:09:50,875 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:09:50,940 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 21:09:55,708 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:09:55,876 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:10:00,733 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:10:00,733 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:10:00,877 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:10:04,749 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 21:10:04,750 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 21:10:05,889 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:10:10,738 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:10:10,894 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:10:15,738 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:10:15,899 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:10:16,513 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 21:10:16,514 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 21:10:16,515 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 21:10:16,518 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 21:10:16,964 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 21:10:17,503 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 21:10:18,965 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 21:10:19,749 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 21:10:19,750 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 21:10:21,880 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:10:24,919 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:10:24,920 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:10:26,880 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:10:29,920 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:10:29,920 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:10:31,881 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:10:34,750 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 21:10:34,750 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 21:10:36,901 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:10:39,922 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:10:39,922 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:10:41,902 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:10:44,930 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:10:46,912 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:10:47,504 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 21:10:48,579 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 21:10:48,581 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 21:10:48,581 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 21:10:48,584 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 21:10:48,984 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 21:10:49,750 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 21:10:49,750 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 21:10:50,985 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 21:10:52,906 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:10:54,981 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:10:57,907 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:10:59,981 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:11:02,908 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:11:04,750 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 21:11:04,750 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 21:11:08,877 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:11:09,997 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:11:09,997 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:11:13,877 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:11:15,002 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:11:15,007 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:11:17,515 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 21:11:19,516 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:11:19,753 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 21:11:19,753 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 21:11:20,167 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 21:11:20,169 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 21:11:20,170 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 21:11:20,175 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 21:11:21,008 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 21:11:21,008 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 21:11:25,005 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:11:25,176 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:11:30,006 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:11:30,177 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:11:34,827 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 21:11:34,827 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 21:11:35,996 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:11:40,007 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:11:40,997 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:11:45,008 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:11:45,997 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:11:47,522 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 21:11:49,836 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 21:11:49,842 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 21:11:51,944 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:11:53,033 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 21:11:53,035 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 21:11:53,036 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 21:11:53,041 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 21:11:53,042 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 21:11:55,014 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:11:55,044 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 21:11:57,042 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:12:00,015 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:12:02,043 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:12:04,830 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 21:12:04,831 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 21:12:07,975 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:12:10,017 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:12:12,976 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:12:15,017 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:12:17,535 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 21:12:18,536 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:12:19,837 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 21:12:19,838 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 21:12:22,648 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 21:12:22,650 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 21:12:22,651 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 21:12:22,653 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 21:12:23,067 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 21:12:23,654 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:12:25,052 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:12:25,069 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 21:12:28,654 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:12:30,053 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:12:33,655 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:12:34,834 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 21:12:34,835 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 21:12:39,187 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:12:40,154 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:12:44,187 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:12:45,163 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:12:47,523 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 21:12:49,530 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:12:49,837 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 21:12:49,837 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 21:12:50,334 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 21:12:50,335 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 21:12:50,335 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 21:12:50,337 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 21:12:51,093 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 21:12:53,094 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 21:12:55,161 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:12:55,338 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:13:00,161 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:13:00,338 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:13:04,837 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 21:13:04,838 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 21:13:05,951 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:13:10,163 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:13:10,951 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:13:15,171 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:13:15,961 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:13:17,525 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 21:13:17,575 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 21:13:17,576 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 21:13:17,577 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 21:13:17,578 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 21:13:18,116 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 21:13:19,117 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 21:13:19,837 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 21:13:19,838 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 21:13:20,969 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:13:25,172 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:13:25,970 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:13:30,173 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:13:30,970 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:13:34,837 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 21:13:34,838 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 21:13:36,948 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:13:40,174 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:13:41,956 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:13:45,184 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:13:46,954 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:13:47,353 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 21:13:47,354 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 21:13:47,354 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 21:13:47,356 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 21:13:47,529 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 21:13:48,144 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 21:13:49,145 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 21:13:49,837 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 21:13:49,838 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 21:13:51,975 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:13:55,185 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:13:56,976 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:14:00,186 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:14:01,976 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:14:04,837 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 21:14:04,838 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 21:14:07,930 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:14:10,188 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:14:12,934 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:14:15,200 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:14:17,011 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 21:14:17,013 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 21:14:17,013 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 21:14:17,015 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 21:14:17,169 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 21:14:17,533 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 21:14:18,535 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:14:19,170 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 21:14:19,838 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 21:14:19,838 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 21:14:23,948 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:14:25,418 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:14:28,948 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:14:30,419 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:14:30,419 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:14:33,949 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:14:34,838 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 21:14:34,838 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 21:14:39,943 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:14:40,420 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:14:40,420 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:14:44,952 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:14:45,427 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:14:46,993 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 21:14:46,995 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 21:14:46,995 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 21:14:46,997 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 21:14:47,190 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 21:14:47,538 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 21:14:49,192 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 21:14:49,838 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 21:14:49,838 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 21:14:50,931 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:14:55,426 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:14:55,931 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:15:00,427 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:15:00,932 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:15:04,838 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 21:15:04,839 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 21:15:05,965 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:15:10,429 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:15:10,965 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:15:15,436 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:15:15,975 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:15:17,542 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 21:15:19,343 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 21:15:19,344 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 21:15:19,344 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 21:15:19,345 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 21:15:19,838 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 21:15:19,839 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 21:15:20,220 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 21:15:21,221 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 21:15:21,924 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:15:25,492 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:15:25,492 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:15:26,925 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:15:30,493 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:15:30,493 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:15:31,926 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:15:34,838 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 21:15:34,839 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 21:15:37,911 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:15:40,718 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:15:40,718 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:15:42,911 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:15:45,723 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:15:47,546 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 21:15:48,551 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:15:49,845 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 21:15:49,846 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 21:15:51,049 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 21:15:51,051 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 21:15:51,052 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 21:15:51,053 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 21:15:51,246 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 21:15:53,248 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 21:15:54,054 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:15:55,725 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:15:59,055 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:16:00,726 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:16:00,726 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:16:04,056 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:16:04,843 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 21:16:04,844 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 21:16:10,005 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:16:10,728 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:16:10,728 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:16:15,012 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:16:15,730 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:16:17,550 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 21:16:19,846 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 21:16:19,846 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 21:16:20,439 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 21:16:20,441 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 21:16:20,442 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 21:16:20,442 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:16:20,444 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 21:16:21,271 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 21:16:21,272 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 21:16:25,445 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:16:25,845 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:16:25,845 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:16:30,446 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:16:30,846 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:16:30,846 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:16:34,846 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 21:16:34,847 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 21:16:35,937 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:16:39,848 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:16:39,848 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:16:40,938 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:16:44,850 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:16:45,940 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:16:47,559 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 21:16:49,023 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 21:16:49,024 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 21:16:49,025 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 21:16:49,026 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 21:16:49,298 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 21:16:49,846 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 21:16:49,847 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 21:16:51,300 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 21:16:51,928 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:16:54,857 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:16:56,929 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:16:59,857 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:17:01,929 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:17:04,846 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 21:17:04,847 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 21:17:07,017 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:17:09,859 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:17:12,017 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:17:14,859 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:17:17,018 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:17:17,560 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 21:17:19,857 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 21:17:19,858 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 21:17:22,988 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:17:23,845 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 21:17:23,846 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 21:17:23,846 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 21:17:23,849 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 21:17:24,331 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 21:17:24,990 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:17:25,332 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 21:17:28,850 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:17:29,990 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:17:29,991 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:17:33,850 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:17:34,852 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 21:17:34,852 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 21:17:38,951 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:17:39,992 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:17:39,992 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:17:43,952 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:17:44,993 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:17:44,993 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:17:47,570 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 21:17:49,574 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:17:49,866 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 21:17:49,867 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 21:17:52,835 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 21:17:52,836 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 21:17:52,837 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 21:17:52,839 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 21:17:53,355 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 21:17:54,840 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:17:55,238 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:17:55,238 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:17:55,357 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 21:17:59,840 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:18:00,239 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:18:00,239 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:18:04,841 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:18:04,855 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 21:18:04,856 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 21:18:09,963 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:18:10,241 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:18:14,963 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:18:15,241 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:18:15,242 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:18:17,570 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 21:18:19,859 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 21:18:19,863 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 21:18:20,927 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:18:22,225 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 21:18:22,227 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 21:18:22,227 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 21:18:22,229 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 21:18:22,381 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 21:18:23,382 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 21:18:25,378 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:18:25,379 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:18:26,230 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:18:30,379 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:18:30,379 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:18:31,231 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:18:34,856 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 21:18:34,857 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 21:18:36,982 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:18:40,387 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:18:41,982 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:18:45,388 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:18:46,985 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:18:47,571 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 21:18:49,866 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 21:18:49,870 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 21:18:52,024 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:18:52,081 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 21:18:52,082 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 21:18:52,082 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 21:18:52,084 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 21:18:52,411 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 21:18:53,412 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 21:18:55,488 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:18:55,489 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:18:57,085 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:19:00,489 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:19:02,085 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:19:04,860 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 21:19:04,861 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 21:19:07,972 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:19:10,491 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:19:12,972 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:19:15,492 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:19:15,492 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:19:17,575 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 21:19:18,577 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:19:19,864 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 21:19:19,868 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 21:19:23,971 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:19:24,449 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 21:19:24,451 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 21:19:24,451 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 21:19:24,452 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 21:19:25,437 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 21:19:25,437 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 21:19:25,600 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:19:29,453 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:19:30,602 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:19:30,602 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:19:34,454 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:19:34,864 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 21:19:34,864 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 21:19:39,940 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:19:40,604 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:19:40,604 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:19:44,941 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:19:45,605 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:19:45,605 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:19:47,579 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 21:19:49,868 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 21:19:49,869 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 21:19:50,002 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:19:54,924 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 21:19:54,926 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 21:19:54,926 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 21:19:54,928 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 21:19:55,462 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 21:19:55,610 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:19:55,929 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:19:57,464 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 21:20:00,612 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:20:00,930 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:20:04,867 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 21:20:04,868 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 21:20:06,002 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:20:10,636 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:20:10,637 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:20:11,003 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:20:15,637 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:20:15,637 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:20:16,004 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:20:17,584 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 21:20:19,877 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 21:20:19,881 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 21:20:22,003 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:20:23,997 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 21:20:23,999 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 21:20:23,999 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 21:20:24,001 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 21:20:24,487 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 21:20:25,488 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 21:20:25,643 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:20:28,002 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:20:30,644 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:20:33,002 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:20:34,871 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 21:20:34,872 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 21:20:38,945 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:20:40,768 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:20:43,945 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:20:45,769 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:20:47,585 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 21:20:49,590 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:20:49,880 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 21:20:49,881 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 21:20:54,636 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 21:20:54,637 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 21:20:54,638 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 21:20:54,638 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:20:54,640 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 21:20:55,040 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:20:55,041 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:20:55,517 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 21:20:55,518 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 21:20:59,641 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:21:00,041 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:21:04,641 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:21:04,874 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 21:21:04,874 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 21:21:09,999 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:21:10,043 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:21:14,999 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:21:15,043 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:21:17,588 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 21:21:19,882 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 21:21:19,885 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 21:21:20,039 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:21:25,043 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:21:25,055 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:21:25,055 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:21:25,288 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 21:21:25,289 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 21:21:25,289 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 21:21:25,291 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 21:21:25,537 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 21:21:27,539 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 21:21:30,056 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:21:30,292 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:21:34,876 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 21:21:34,877 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 21:21:36,046 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:21:40,057 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:21:41,047 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:21:45,058 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:21:46,047 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:21:47,592 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 21:21:49,882 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 21:21:49,883 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 21:21:51,995 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:21:54,334 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 21:21:54,336 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 21:21:54,336 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 21:21:54,338 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 21:21:54,566 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 21:21:55,067 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:21:55,567 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 21:21:57,339 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:22:00,067 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:22:02,340 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:22:04,879 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 21:22:04,880 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 21:22:07,984 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:22:10,113 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:22:12,984 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:22:15,114 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:22:17,596 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 21:22:18,601 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:22:19,892 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 21:22:19,893 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 21:22:23,705 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 21:22:23,707 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 21:22:23,707 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 21:22:23,708 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:22:23,709 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 21:22:24,593 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 21:22:25,137 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:22:25,594 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 21:22:28,710 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:22:30,137 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:22:33,710 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:22:34,883 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 21:22:34,884 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 21:22:39,007 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:22:40,138 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:22:44,007 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:22:45,139 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:22:47,620 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 21:22:49,625 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:22:49,891 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 21:22:49,894 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 21:22:52,700 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 21:22:52,701 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 21:22:52,701 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 21:22:52,703 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 21:22:53,619 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 21:22:53,620 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 21:22:54,704 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:22:55,147 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:22:59,705 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:23:00,147 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:23:04,705 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:23:04,889 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 21:23:04,890 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 21:23:10,004 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:23:10,149 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:23:15,005 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:23:15,149 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:23:17,612 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 21:23:19,901 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 21:23:19,902 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 21:23:20,028 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:23:23,041 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 21:23:23,043 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 21:23:23,044 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 21:23:23,049 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 21:23:23,650 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 21:23:23,650 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 21:23:25,050 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:23:25,186 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:23:30,050 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:23:30,187 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:23:34,896 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 21:23:34,897 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 21:23:35,064 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:23:40,065 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:23:40,189 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:23:45,065 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:23:45,189 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:23:47,607 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 21:23:49,905 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 21:23:49,910 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 21:23:51,038 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:23:53,962 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 21:23:53,964 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 21:23:53,964 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 21:23:53,968 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 21:23:54,681 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 21:23:55,366 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:23:55,682 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 21:23:56,969 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:24:00,367 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:24:01,970 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:24:04,900 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 21:24:04,901 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 21:24:07,070 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:24:10,369 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:24:12,070 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:24:15,369 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:24:17,071 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:24:17,611 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 21:24:19,900 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 21:24:19,901 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 21:24:23,026 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:24:25,376 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:24:26,387 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 21:24:26,388 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 21:24:26,388 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 21:24:26,390 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 21:24:26,708 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 21:24:27,709 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 21:24:28,391 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:24:30,375 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:24:33,391 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:24:34,900 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 21:24:34,901 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 21:24:39,076 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:24:40,376 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:24:44,076 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:24:45,377 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:24:47,615 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 21:24:49,617 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:24:49,900 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 21:24:49,901 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 21:24:55,034 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:24:55,381 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:24:57,639 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 21:24:57,641 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 21:24:57,642 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 21:24:57,647 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 21:24:57,735 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 21:24:59,737 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 21:25:00,382 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:25:00,648 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:25:04,901 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 21:25:04,901 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 21:25:05,973 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:25:10,383 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:25:10,974 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:25:15,384 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:25:15,974 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:25:17,618 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 21:25:19,901 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 21:25:19,901 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 21:25:21,045 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:25:25,394 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:25:26,052 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:25:28,550 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 21:25:28,551 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 21:25:28,552 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 21:25:28,557 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 21:25:28,773 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 21:25:29,774 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 21:25:30,391 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:25:31,558 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:25:34,901 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 21:25:34,901 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 21:25:37,060 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:25:40,393 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:25:42,061 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:25:45,393 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:25:47,061 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:25:47,623 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 21:25:49,901 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 21:25:49,901 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 21:25:53,052 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:25:55,395 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:25:58,068 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:26:00,406 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:26:01,782 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 21:26:01,784 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 21:26:01,784 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 21:26:01,787 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 21:26:01,802 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 21:26:03,788 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:26:03,804 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 21:26:04,901 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 21:26:04,902 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 21:26:09,002 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:26:10,402 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:26:14,003 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:26:15,402 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:26:17,626 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 21:26:19,629 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:26:19,901 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 21:26:19,901 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 21:26:25,003 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:26:25,404 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:26:30,007 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:26:30,410 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:26:33,950 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 21:26:33,951 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 21:26:33,952 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 21:26:33,955 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 21:26:34,832 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 21:26:34,901 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 21:26:34,902 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 21:26:35,030 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:26:35,833 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 21:26:40,031 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:26:40,411 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:26:45,031 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:26:45,412 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:26:47,631 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 21:26:49,901 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 21:26:49,902 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 21:26:50,985 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:26:55,413 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:26:55,986 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:27:00,429 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:27:00,995 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:27:03,736 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 21:27:03,738 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 21:27:03,739 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 21:27:03,740 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 21:27:03,865 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 21:27:04,902 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 21:27:04,902 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 21:27:06,079 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:27:06,868 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 21:27:10,423 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:27:11,080 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:27:15,424 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:27:16,080 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:27:17,634 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 21:27:19,902 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 21:27:19,902 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 21:27:22,076 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:27:25,558 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:27:27,077 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:27:30,559 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:27:32,080 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:27:34,911 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 21:27:34,913 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 21:27:37,011 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 21:27:37,013 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 21:27:37,013 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 21:27:37,019 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 21:27:37,899 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 21:27:38,019 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:27:38,901 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 21:27:40,569 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:27:43,020 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:27:45,569 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:27:47,639 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 21:27:48,641 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:27:50,021 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 21:27:50,022 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 21:27:54,160 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:27:55,570 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:27:59,160 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:28:00,571 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:28:04,171 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:28:04,934 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 21:28:04,935 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 21:28:07,425 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 21:28:07,426 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 21:28:07,426 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 21:28:07,429 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 21:28:07,933 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 21:28:08,935 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 21:28:09,430 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:28:10,830 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:28:10,830 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:28:14,430 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:28:15,831 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:28:15,831 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:28:17,643 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 21:28:19,646 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:28:19,934 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 21:28:19,934 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 21:28:25,070 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:28:25,832 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:28:25,833 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:28:30,071 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:28:30,833 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:28:30,833 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:28:34,942 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 21:28:34,950 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 21:28:36,038 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:28:39,041 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 21:28:39,043 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 21:28:39,044 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 21:28:39,046 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 21:28:39,960 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 21:28:40,839 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:28:40,960 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 21:28:41,047 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:28:45,839 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:28:46,048 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:28:47,647 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 21:28:49,937 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 21:28:49,938 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 21:28:52,020 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:28:55,841 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:28:57,021 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:29:00,841 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:29:02,022 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:29:04,945 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 21:29:04,950 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 21:29:07,120 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:29:10,474 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 21:29:10,475 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 21:29:10,476 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 21:29:10,480 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 21:29:10,992 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 21:29:11,584 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:29:12,481 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:29:12,993 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 21:29:16,586 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:29:16,586 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:29:17,482 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:29:17,651 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 21:29:19,942 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 21:29:19,943 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 21:29:23,069 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:29:25,587 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:29:25,588 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:29:28,069 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:29:30,588 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:29:30,589 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:29:33,070 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:29:34,942 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 21:29:34,943 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 21:29:39,044 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:29:40,600 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:29:42,453 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 21:29:42,454 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 21:29:42,454 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 21:29:42,457 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 21:29:43,017 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 21:29:44,458 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:29:45,019 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 21:29:45,601 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:29:47,655 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 21:29:49,657 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:29:49,942 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 21:29:49,943 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 21:29:55,012 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:29:55,149 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:30:00,013 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:30:00,149 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:30:04,942 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 21:30:04,943 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 21:30:06,044 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:30:10,022 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:30:11,047 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:30:15,025 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:30:15,809 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 21:30:15,810 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 21:30:15,811 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 21:30:15,816 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 21:30:16,051 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 21:30:16,817 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:30:17,659 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 21:30:19,053 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 21:30:19,943 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 21:30:19,943 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 21:30:22,090 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:30:25,253 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:30:27,090 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:30:30,254 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:30:32,091 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:30:34,943 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 21:30:34,943 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 21:30:38,050 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:30:40,255 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:30:43,050 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:30:45,267 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:30:47,663 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 21:30:48,046 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 21:30:48,048 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 21:30:48,048 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 21:30:48,049 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 21:30:48,079 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 21:30:49,050 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:30:49,943 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 21:30:49,943 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 21:30:51,081 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 21:30:55,032 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:30:55,481 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:31:00,033 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:31:00,482 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:31:04,943 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 21:31:04,944 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 21:31:05,062 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:31:10,062 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:31:10,548 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:31:15,067 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:31:15,554 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:31:17,661 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 21:31:19,946 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 21:31:19,949 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 21:31:20,268 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 21:31:20,269 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 21:31:20,269 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 21:31:20,271 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:31:20,272 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 21:31:21,107 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 21:31:23,108 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 21:31:25,273 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:31:25,743 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:31:25,743 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:31:30,274 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:31:30,743 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:31:35,001 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 21:31:35,001 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 21:31:36,127 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:31:40,745 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:31:40,745 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:31:41,127 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:31:45,746 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:31:45,746 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:31:46,128 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:31:47,666 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 21:31:50,006 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 21:31:50,007 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 21:31:51,130 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:31:53,198 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 21:31:53,200 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 21:31:53,201 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 21:31:53,206 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 21:31:54,131 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 21:31:55,132 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 21:31:55,753 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:31:56,207 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:32:00,753 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:32:01,208 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:32:05,003 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 21:32:05,004 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 21:32:07,076 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:32:10,755 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:32:12,076 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:32:15,755 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:32:17,077 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:32:17,670 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 21:32:20,003 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 21:32:20,003 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 21:32:22,152 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:32:24,762 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 21:32:24,764 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 21:32:24,764 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 21:32:24,766 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 21:32:25,161 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 21:32:25,763 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:32:27,162 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 21:32:27,767 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:32:30,764 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:32:32,768 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:32:35,003 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 21:32:35,004 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 21:32:38,141 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:32:40,810 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:32:43,141 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:32:45,811 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:32:47,674 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 21:32:48,676 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:32:50,013 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 21:32:50,021 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 21:32:54,196 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:32:55,497 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 21:32:55,499 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 21:32:55,499 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 21:32:55,505 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 21:32:55,935 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:32:55,936 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:32:56,191 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 21:32:57,192 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 21:32:59,506 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:33:00,936 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:33:00,937 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:33:04,507 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:33:05,006 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 21:33:05,006 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 21:33:10,170 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:33:10,938 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:33:10,938 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:33:15,171 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:33:15,939 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:33:15,939 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:33:17,678 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 21:33:20,006 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 21:33:20,007 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 21:33:21,169 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:33:25,948 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:33:26,174 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:33:28,011 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 21:33:28,013 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 21:33:28,014 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 21:33:28,016 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 21:33:28,218 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 21:33:29,219 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 21:33:30,946 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:33:32,017 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:33:35,006 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 21:33:35,007 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 21:33:37,099 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:33:40,026 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:33:40,116 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:33:42,099 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:33:45,117 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:33:45,117 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:33:47,100 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:33:47,683 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 21:33:50,006 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 21:33:50,007 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 21:33:52,136 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:33:55,130 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:33:56,138 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 21:33:56,139 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 21:33:56,140 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 21:33:56,142 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 21:33:56,245 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 21:33:57,143 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:33:59,247 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 21:34:00,128 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:34:02,143 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:34:05,006 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 21:34:05,007 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 21:34:08,108 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:34:10,129 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:34:13,108 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:34:15,130 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:34:17,688 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 21:34:18,694 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:34:20,016 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 21:34:20,020 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 21:34:22,676 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 21:34:22,679 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 21:34:22,679 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 21:34:22,681 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 21:34:23,271 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 21:34:24,682 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:34:25,138 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:34:25,138 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:34:25,273 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 21:34:29,682 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:34:30,139 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:34:30,140 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:34:34,683 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:34:35,012 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 21:34:35,012 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 21:34:40,141 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:34:40,141 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:34:40,147 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:34:45,142 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:34:45,142 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:34:45,147 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:34:47,690 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 21:34:50,022 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 21:34:50,028 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 21:34:50,152 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:34:53,673 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 21:34:53,674 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 21:34:53,675 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 21:34:53,676 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 21:34:54,302 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 21:34:55,151 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:34:55,152 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:34:55,303 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 21:34:55,678 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:35:00,152 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:35:00,152 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:35:00,678 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:35:05,014 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 21:35:05,015 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 21:35:06,100 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:35:10,154 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:35:11,100 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:35:15,154 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:35:15,155 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:35:16,101 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:35:17,694 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 21:35:20,020 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 21:35:20,020 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 21:35:21,144 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:35:24,826 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 21:35:24,828 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 21:35:24,828 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 21:35:24,829 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 21:35:25,231 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:35:25,334 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 21:35:26,830 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:35:27,335 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 21:35:30,232 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:35:31,830 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:35:35,018 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 21:35:35,018 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 21:35:37,104 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:35:40,233 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:35:40,234 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:35:42,105 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:35:45,235 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:35:45,235 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:35:47,106 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:35:47,698 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 21:35:50,018 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 21:35:50,019 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 21:35:52,195 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:35:55,244 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:35:56,993 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 21:35:56,994 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 21:35:56,994 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 21:35:56,995 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 21:35:57,358 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 21:35:57,996 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:35:59,360 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 21:36:00,405 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:36:00,405 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:36:02,997 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:36:05,018 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 21:36:05,019 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 21:36:08,128 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:36:10,621 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:36:10,622 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:36:13,129 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:36:15,622 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:36:15,623 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:36:17,703 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 21:36:18,705 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:36:20,026 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 21:36:20,032 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 21:36:24,183 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:36:25,301 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 21:36:25,303 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 21:36:25,303 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 21:36:25,304 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 21:36:25,388 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 21:36:25,706 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:36:25,706 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:36:27,389 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 21:36:29,305 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:36:30,707 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:36:30,707 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:36:34,306 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:36:35,020 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 21:36:35,021 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 21:36:40,113 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:36:40,708 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:36:40,709 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:36:45,113 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:36:45,709 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:36:45,709 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:36:47,706 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 21:36:50,020 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 21:36:50,021 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 21:36:50,187 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:36:55,191 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:36:55,714 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:36:55,795 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 21:36:55,796 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 21:36:55,797 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 21:36:55,799 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 21:36:56,421 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 21:36:57,422 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 21:37:00,715 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:37:00,800 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:37:05,020 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 21:37:05,021 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 21:37:06,099 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:37:10,717 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:37:11,100 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:37:15,717 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:37:16,100 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:37:17,710 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 21:37:20,020 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 21:37:20,021 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 21:37:21,187 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:37:25,726 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:37:26,197 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:37:28,067 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 21:37:28,069 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 21:37:28,069 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 21:37:28,071 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 21:37:28,446 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 21:37:29,447 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 21:37:30,723 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:37:32,071 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:37:35,021 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 21:37:35,021 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 21:37:37,190 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:37:40,724 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:37:42,190 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:37:45,747 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:37:45,747 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:37:47,191 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:37:47,714 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 21:37:50,021 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 21:37:50,021 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 21:37:53,149 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:37:55,757 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:37:58,157 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:38:00,757 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:38:00,758 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:38:00,913 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 21:38:00,915 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 21:38:00,916 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 21:38:00,918 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 21:38:01,473 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 21:38:01,474 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 21:38:03,918 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:38:05,021 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 21:38:05,021 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 21:38:09,121 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:38:10,759 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:38:10,759 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:38:14,121 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:38:15,799 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:38:17,717 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 21:38:19,719 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:38:20,021 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 21:38:20,021 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 21:38:25,154 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:38:25,800 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:38:30,162 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:38:30,810 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:38:31,865 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 21:38:31,867 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 21:38:31,867 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 21:38:31,868 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 21:38:32,509 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 21:38:33,509 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 21:38:35,022 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 21:38:35,023 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 21:38:36,100 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:38:40,808 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:38:41,100 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:38:45,809 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:38:46,101 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:38:47,721 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 21:38:50,022 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 21:38:50,022 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 21:38:51,116 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:38:55,810 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:38:56,116 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:39:00,822 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:39:01,126 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:39:02,499 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 21:39:02,501 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 21:39:02,501 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 21:39:02,503 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 21:39:02,534 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 21:39:03,536 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 21:39:05,022 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 21:39:05,022 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 21:39:06,182 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:39:10,902 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:39:11,182 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:39:15,903 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:39:15,903 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:39:16,183 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:39:17,725 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 21:39:20,022 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 21:39:20,022 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 21:39:22,161 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:39:25,906 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:39:27,165 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:39:30,909 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:39:31,580 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 21:39:31,582 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 21:39:31,583 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 21:39:31,584 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 21:39:32,571 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 21:39:32,585 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:39:33,572 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 21:39:35,022 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 21:39:35,022 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 21:39:38,105 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:39:40,984 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:39:40,984 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:39:43,106 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:39:45,985 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:39:45,985 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:39:47,729 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 21:39:48,732 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:39:50,022 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 21:39:50,022 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 21:39:54,151 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:39:55,986 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:39:55,987 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:39:59,156 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:40:00,991 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:40:02,625 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 21:40:02,626 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 21:40:02,627 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 21:40:02,628 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 21:40:03,599 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 21:40:03,599 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 21:40:04,630 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:40:05,022 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 21:40:05,023 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 21:40:10,037 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:40:10,037 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:40:10,094 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:40:15,038 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:40:15,038 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:40:15,095 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:40:17,733 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 21:40:20,022 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 21:40:20,023 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 21:40:20,100 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:40:25,047 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:40:25,105 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:40:27,129 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 21:40:27,131 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 21:40:27,131 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 21:40:27,133 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 21:40:27,626 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 21:40:27,627 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 21:40:30,046 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:40:30,134 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:40:35,022 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 21:40:35,023 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 21:40:36,109 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:40:40,047 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:40:41,109 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:40:45,048 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:40:46,110 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:40:47,737 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 21:40:50,023 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 21:40:50,024 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 21:40:51,189 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:40:55,059 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:40:55,959 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 21:40:55,960 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 21:40:55,960 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 21:40:55,962 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 21:40:56,652 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 21:40:56,962 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:40:57,653 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 21:41:00,363 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:41:00,363 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:41:01,963 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:41:05,023 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 21:41:05,023 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 21:41:07,121 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:41:10,365 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:41:10,365 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:41:12,122 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:41:15,366 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:41:15,366 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:41:17,123 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:41:17,740 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 21:41:20,023 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 21:41:20,024 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 21:41:22,180 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:41:25,377 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:41:26,008 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 21:41:26,009 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 21:41:26,009 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 21:41:26,011 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 21:41:26,685 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 21:41:27,686 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 21:41:28,011 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:41:30,414 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:41:30,415 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:41:33,012 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:41:35,023 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 21:41:35,023 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 21:41:38,146 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:41:40,416 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:41:40,416 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:41:43,146 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:41:45,417 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:41:45,417 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:41:47,746 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 21:41:48,751 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:41:50,026 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 21:41:50,026 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 21:41:53,619 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 21:41:53,620 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 21:41:53,620 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 21:41:53,622 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 21:41:53,717 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 21:41:54,622 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:41:55,425 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:41:55,718 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 21:41:59,623 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:42:00,426 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:42:00,426 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:42:04,624 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:42:05,026 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 21:42:05,026 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 21:42:10,152 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:42:10,427 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:42:10,427 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:42:15,153 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:42:15,428 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:42:15,428 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:42:17,748 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 21:42:20,035 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 21:42:20,044 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 21:42:21,150 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:42:25,435 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:42:25,635 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 21:42:25,637 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 21:42:25,637 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 21:42:25,638 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 21:42:25,748 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 21:42:26,639 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:42:27,750 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 21:42:30,436 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:42:31,639 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:42:35,030 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 21:42:35,031 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 21:42:37,172 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:42:40,438 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:42:42,173 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:42:45,438 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:42:47,182 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:42:47,752 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 21:42:50,035 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 21:42:50,035 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 21:42:52,195 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:42:52,441 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 21:42:52,443 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 21:42:52,444 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 21:42:52,446 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 21:42:52,778 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 21:42:53,779 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 21:42:55,851 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:42:55,852 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:42:57,447 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:43:00,852 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:43:02,448 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:43:05,032 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 21:43:05,033 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 21:43:08,123 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:43:10,853 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:43:13,124 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:43:15,864 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:43:17,756 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 21:43:18,757 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:43:20,036 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 21:43:20,041 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 21:43:21,512 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 21:43:21,514 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 21:43:21,514 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 21:43:21,516 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 21:43:21,800 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 21:43:23,802 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 21:43:24,517 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:43:25,964 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:43:25,965 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:43:29,518 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:43:30,965 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:43:34,518 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:43:35,036 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 21:43:35,037 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 21:43:40,038 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:43:40,156 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:43:45,038 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:43:45,039 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:43:45,157 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:43:47,762 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 21:43:50,045 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 21:43:50,046 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 21:43:50,230 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:43:53,416 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 21:43:53,417 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 21:43:53,418 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 21:43:53,420 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 21:43:53,830 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 21:43:53,831 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 21:43:55,042 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:43:55,421 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:44:00,043 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:44:00,421 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:44:05,039 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 21:44:05,040 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 21:44:06,149 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:44:10,044 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:44:11,149 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:44:15,045 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:44:16,150 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:44:17,763 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 21:44:20,039 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 21:44:20,040 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 21:44:22,146 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:44:25,049 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:44:27,147 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:44:27,477 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 21:44:27,480 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 21:44:27,480 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 21:44:27,482 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 21:44:27,866 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 21:44:27,867 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 21:44:30,050 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:44:32,484 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:44:35,039 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 21:44:35,040 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 21:44:38,162 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:44:40,051 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:44:43,163 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:44:45,052 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:44:47,767 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 21:44:48,769 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:44:50,039 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 21:44:50,040 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 21:44:54,174 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:44:55,066 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:44:56,793 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 21:44:56,793 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 21:44:56,794 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 21:44:56,795 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 21:44:56,891 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 21:44:57,893 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 21:44:59,797 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:45:00,192 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:45:00,192 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:45:04,797 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:45:05,040 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 21:45:05,040 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 21:45:10,206 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:45:10,289 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:45:10,289 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:45:15,207 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:45:15,290 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:45:15,290 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:45:17,771 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 21:45:20,040 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 21:45:20,040 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 21:45:21,210 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:45:25,305 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:45:26,210 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:45:26,619 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 21:45:26,620 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 21:45:26,620 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 21:45:26,621 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 21:45:26,924 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 21:45:27,925 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 21:45:30,301 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:45:31,622 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:45:35,040 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 21:45:35,040 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 21:45:37,205 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:45:40,302 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:45:42,206 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:45:45,303 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:45:47,206 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:45:47,775 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 21:45:50,040 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 21:45:50,040 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 21:45:53,149 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:45:55,313 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:45:58,153 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:45:58,310 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 21:45:58,312 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 21:45:58,313 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 21:45:58,315 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 21:45:58,953 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 21:45:59,954 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 21:46:00,311 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:46:03,316 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:46:05,040 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 21:46:05,041 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 21:46:09,205 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:46:10,312 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:46:14,205 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:46:15,313 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:46:17,779 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 21:46:19,781 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:46:20,040 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 21:46:20,040 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 21:46:25,204 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:46:25,324 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:46:28,587 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 21:46:28,589 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 21:46:28,589 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 21:46:28,590 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 21:46:28,989 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 21:46:29,990 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 21:46:30,319 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:46:30,591 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:46:35,040 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 21:46:35,041 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 21:46:36,202 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:46:40,321 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:46:41,204 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:46:45,322 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:46:46,205 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:46:47,783 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 21:46:50,040 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 21:46:50,041 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 21:46:52,168 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:46:55,328 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:46:57,172 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:46:57,561 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 21:46:57,562 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 21:46:57,562 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 21:46:57,563 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 21:46:58,017 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 21:46:58,017 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 21:47:00,334 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:47:02,564 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:47:05,041 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 21:47:05,041 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 21:47:08,131 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:47:10,335 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:47:13,132 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:47:15,336 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:47:17,788 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 21:47:18,790 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:47:20,040 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 21:47:20,041 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 21:47:24,168 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:47:25,346 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:47:26,636 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 21:47:26,638 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 21:47:26,638 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 21:47:26,639 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 21:47:27,047 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 21:47:28,048 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 21:47:29,640 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:47:30,347 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:47:34,640 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:47:35,041 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 21:47:35,041 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 21:47:40,142 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:47:40,348 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:47:45,143 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:47:45,349 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:47:47,791 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 21:47:50,044 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 21:47:50,044 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 21:47:50,191 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:47:55,199 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:47:55,258 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 21:47:55,259 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 21:47:55,260 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 21:47:55,261 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 21:47:55,403 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:47:56,076 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 21:47:56,077 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 21:48:00,262 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:48:00,666 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:48:05,041 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 21:48:05,041 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 21:48:06,139 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:48:10,667 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:48:11,140 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:48:15,668 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:48:16,141 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:48:17,795 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 21:48:20,041 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 21:48:20,041 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 21:48:22,129 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:48:25,673 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:48:26,781 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 21:48:26,782 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 21:48:26,783 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 21:48:26,784 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 21:48:27,113 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 21:48:27,785 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:48:28,114 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 21:48:30,674 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:48:32,786 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:48:35,041 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 21:48:35,042 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 21:48:38,156 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:48:40,676 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:48:43,157 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:48:45,676 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:48:47,799 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 21:48:48,801 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:48:50,041 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 21:48:50,042 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 21:48:54,209 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:48:55,682 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:48:55,682 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:48:58,365 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 21:48:58,366 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 21:48:58,367 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 21:48:58,368 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 21:48:59,141 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 21:48:59,369 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:49:00,142 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 21:49:00,766 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:49:04,370 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:49:05,041 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 21:49:05,042 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 21:49:10,143 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:49:10,768 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:49:15,143 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:49:15,802 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:49:15,802 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:49:17,803 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 21:49:20,041 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 21:49:20,042 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 21:49:21,121 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:49:25,812 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:49:26,127 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:49:28,774 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 21:49:28,776 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 21:49:28,777 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 21:49:28,778 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 21:49:29,171 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 21:49:30,172 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 21:49:31,174 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:49:31,175 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:49:31,779 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:49:35,042 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 21:49:35,042 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 21:49:37,163 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:49:40,176 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:49:40,176 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:49:42,163 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:49:45,179 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:49:45,179 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:49:47,164 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:49:47,808 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 21:49:50,042 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 21:49:50,042 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 21:49:52,199 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:49:55,181 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:49:57,207 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:50:00,195 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:50:00,195 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:50:01,697 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 21:50:01,699 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 21:50:01,699 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 21:50:01,700 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 21:50:02,207 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 21:50:02,207 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 21:50:02,701 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:50:05,042 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 21:50:05,042 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 21:50:08,163 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:50:10,193 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:50:13,163 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:50:15,268 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:50:15,268 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:50:17,811 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 21:50:18,813 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:50:20,042 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 21:50:20,042 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 21:50:24,214 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:50:25,278 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:50:29,017 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 21:50:29,018 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 21:50:29,018 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 21:50:29,019 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 21:50:29,236 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 21:50:30,020 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:50:30,237 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 21:50:30,426 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:50:30,427 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:50:35,021 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:50:35,042 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 21:50:35,042 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 21:50:40,211 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:50:40,428 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:50:40,428 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:50:45,212 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:50:45,429 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:50:45,429 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:50:47,815 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 21:50:50,042 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 21:50:50,043 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 21:50:51,201 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:50:55,435 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:50:56,209 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:50:58,815 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 21:50:58,817 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 21:50:58,817 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 21:50:58,818 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 21:50:59,270 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 21:51:00,271 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 21:51:00,441 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:51:00,442 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:51:01,819 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:51:05,042 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 21:51:05,043 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 21:51:07,203 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:51:10,443 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:51:12,204 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:51:15,444 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:51:17,204 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:51:17,819 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 21:51:20,042 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 21:51:20,043 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 21:51:23,124 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:51:25,452 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:51:25,453 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:51:27,532 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 21:51:27,534 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 21:51:27,534 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 21:51:27,535 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 21:51:28,305 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 21:51:28,305 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 21:51:28,536 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:51:30,451 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:51:30,451 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:51:33,536 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:51:35,043 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 21:51:35,043 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 21:51:39,177 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:51:40,533 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:51:40,620 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:51:44,177 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:51:45,620 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:51:45,621 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:51:47,823 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 21:51:49,826 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:51:50,042 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 21:51:50,043 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 21:51:55,136 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:51:55,628 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:51:57,950 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 21:51:57,951 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 21:51:57,951 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 21:51:57,953 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 21:51:58,335 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 21:51:59,336 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 21:52:00,628 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:52:00,954 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:52:05,043 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 21:52:05,043 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 21:52:06,160 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:52:10,629 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:52:11,161 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:52:15,630 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:52:15,630 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:52:16,161 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:52:17,826 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 21:52:20,053 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 21:52:20,054 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 21:52:22,158 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:52:25,510 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 21:52:25,512 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 21:52:25,512 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 21:52:25,514 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 21:52:25,658 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:52:25,904 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:52:26,367 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 21:52:27,368 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 21:52:27,515 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:52:30,904 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:52:30,905 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:52:32,516 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:52:35,046 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 21:52:35,047 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 21:52:38,211 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:52:41,031 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:52:41,031 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:52:43,212 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:52:46,032 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:52:46,032 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:52:47,831 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 21:52:48,833 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:52:50,046 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 21:52:50,047 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 21:52:54,193 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:52:56,051 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:52:57,351 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 21:52:57,354 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 21:52:57,354 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 21:52:57,356 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 21:52:57,396 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 21:52:59,357 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:52:59,397 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 21:53:01,039 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:53:04,357 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:53:05,046 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 21:53:05,047 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 21:53:10,135 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:53:11,040 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:53:15,136 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:53:16,041 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:53:17,834 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 21:53:20,050 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 21:53:20,051 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 21:53:20,202 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:53:25,050 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:53:25,071 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 21:53:25,073 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 21:53:25,073 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 21:53:25,075 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 21:53:25,429 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 21:53:26,076 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:53:27,431 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 21:53:30,051 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:53:31,077 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:53:35,050 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 21:53:35,051 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 21:53:36,215 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:53:40,051 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:53:41,216 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:53:45,052 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:53:46,216 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:53:47,838 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 21:53:50,050 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 21:53:50,051 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 21:53:52,204 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:53:55,070 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:53:56,504 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 21:53:56,505 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 21:53:56,506 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 21:53:56,507 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 21:53:57,457 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 21:53:57,507 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:53:59,459 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 21:54:00,063 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:54:02,508 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:54:05,050 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 21:54:05,051 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 21:54:08,138 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:54:10,065 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:54:13,138 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:54:15,065 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:54:17,842 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 21:54:18,845 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:54:20,050 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 21:54:20,051 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 21:54:24,208 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:54:25,069 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:54:26,806 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 21:54:26,808 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 21:54:26,808 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 21:54:26,810 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 21:54:27,497 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 21:54:29,499 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 21:54:29,811 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:54:30,250 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:54:30,250 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:54:34,812 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:54:35,050 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 21:54:35,051 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 21:54:40,119 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:54:40,251 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:54:40,252 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:54:45,120 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:54:45,252 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:54:45,252 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:54:47,847 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 21:54:50,051 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 21:54:50,051 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 21:54:50,163 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:54:55,168 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:54:55,261 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:54:57,282 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 21:54:57,284 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 21:54:57,284 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 21:54:57,285 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 21:54:57,531 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 21:54:59,533 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 21:55:00,263 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:55:00,286 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:55:05,051 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 21:55:05,051 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 21:55:06,157 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:55:10,264 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:55:11,158 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:55:15,265 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:55:16,159 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:55:17,851 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 21:55:20,051 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 21:55:20,051 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 21:55:22,135 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:55:25,272 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:55:27,136 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:55:27,496 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 21:55:27,498 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 21:55:27,499 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 21:55:27,500 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 21:55:27,561 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 21:55:29,563 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 21:55:30,277 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:55:30,277 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:55:32,501 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:55:35,051 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 21:55:35,051 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 21:55:38,180 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:55:40,278 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:55:43,181 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:55:45,279 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:55:47,855 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 21:55:48,857 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:55:50,051 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 21:55:50,051 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 21:55:54,197 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:55:55,285 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:55:59,202 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:55:59,252 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 21:55:59,253 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 21:55:59,253 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 21:55:59,254 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 21:55:59,594 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 21:56:00,397 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:56:01,596 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 21:56:04,255 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:56:05,051 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 21:56:05,052 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 21:56:10,141 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:56:10,398 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:56:15,142 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:56:15,399 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:56:15,400 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:56:17,859 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 21:56:20,051 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 21:56:20,052 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 21:56:20,195 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:56:25,201 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:56:25,401 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:56:30,200 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:56:30,404 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:56:30,405 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:56:30,654 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 21:56:30,655 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 21:56:30,656 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 21:56:30,657 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 21:56:31,631 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 21:56:33,632 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 21:56:35,051 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 21:56:35,052 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 21:56:36,146 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:56:40,408 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:56:40,408 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:56:41,146 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:56:45,409 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:56:45,409 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:56:46,147 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:56:47,863 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 21:56:50,051 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 21:56:50,052 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 21:56:52,129 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:56:55,414 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:56:57,143 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:56:59,754 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 21:56:59,756 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 21:56:59,756 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 21:56:59,758 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 21:57:00,421 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:57:00,658 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 21:57:01,659 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 21:57:02,759 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:57:05,052 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 21:57:05,052 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 21:57:08,171 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:57:10,423 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:57:10,423 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:57:13,172 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:57:15,424 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:57:15,424 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:57:17,866 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 21:57:18,869 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:57:20,052 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 21:57:20,052 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 21:57:24,230 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:57:25,436 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:57:28,494 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 21:57:28,496 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 21:57:28,496 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 21:57:28,498 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 21:57:28,685 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 21:57:29,498 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:57:29,686 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 21:57:30,435 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:57:34,499 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:57:35,052 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 21:57:35,052 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 21:57:40,149 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:57:40,436 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:57:45,150 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:57:45,437 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:57:47,872 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 21:57:50,052 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 21:57:50,053 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 21:57:50,206 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:57:55,210 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:57:55,438 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:58:00,212 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:58:00,326 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 21:58:00,328 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 21:58:00,329 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 21:58:00,330 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 21:58:00,472 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:58:00,713 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 21:58:01,714 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 21:58:05,052 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 21:58:05,053 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 21:58:06,156 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:58:10,478 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:58:11,157 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:58:15,540 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:58:16,157 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:58:17,875 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 21:58:20,052 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 21:58:20,053 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 21:58:22,138 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:58:25,542 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:58:27,142 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:58:30,552 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:58:31,568 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 21:58:31,570 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 21:58:31,570 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 21:58:31,571 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 21:58:31,741 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 21:58:32,572 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:58:33,742 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 21:58:35,052 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 21:58:35,053 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 21:58:38,191 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:58:40,552 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:58:43,192 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:58:45,553 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:58:47,878 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 21:58:48,881 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:58:50,052 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 21:58:50,053 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 21:58:54,120 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:58:55,554 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:58:59,120 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:59:00,561 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:59:04,076 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 21:59:04,078 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 21:59:04,078 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 21:59:04,080 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 21:59:04,768 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 21:59:05,053 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 21:59:05,053 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 21:59:05,170 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:59:05,769 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 21:59:10,171 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:59:10,564 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:59:15,172 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:59:15,565 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:59:17,882 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 21:59:20,053 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 21:59:20,053 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 21:59:20,218 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:59:25,219 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:59:25,566 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:59:30,227 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:59:30,576 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:59:34,319 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 21:59:34,321 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 21:59:34,321 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 21:59:34,322 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 21:59:34,795 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 21:59:35,053 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 21:59:35,054 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 21:59:35,796 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 21:59:36,190 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:59:40,719 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:59:41,190 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:59:45,719 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:59:46,191 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:59:47,887 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 21:59:50,053 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 21:59:50,054 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 21:59:51,195 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 21:59:55,053 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 21:59:56,195 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:00:00,060 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:00:01,199 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:00:05,053 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 22:00:05,053 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 22:00:05,399 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 22:00:05,401 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 22:00:05,402 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 22:00:05,403 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 22:00:05,824 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 22:00:06,404 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:00:07,826 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 22:00:10,065 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:00:11,405 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:00:15,066 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:00:16,405 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:00:17,891 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 22:00:20,053 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 22:00:20,054 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 22:00:22,139 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:00:25,067 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:00:27,146 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:00:30,076 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:00:32,154 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:00:32,677 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 22:00:32,679 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 22:00:32,679 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 22:00:32,681 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 22:00:32,850 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 22:00:33,852 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 22:00:35,053 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 22:00:35,054 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 22:00:37,165 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:00:40,085 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:00:42,166 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:00:45,086 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:00:47,166 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:00:47,895 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 22:00:50,053 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 22:00:50,054 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 22:00:52,224 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:00:55,087 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:00:57,225 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:01:00,094 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:01:02,230 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:01:03,316 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 22:01:03,318 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 22:01:03,319 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 22:01:03,320 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 22:01:03,881 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 22:01:05,053 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 22:01:05,054 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 22:01:05,882 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 22:01:08,143 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:01:10,094 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:01:13,143 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:01:15,095 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:01:17,899 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 22:01:18,901 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:01:20,054 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 22:01:20,054 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 22:01:24,199 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:01:25,141 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:01:25,142 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:01:29,202 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:01:30,150 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:01:34,205 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:01:34,443 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 22:01:34,445 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 22:01:34,446 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 22:01:34,447 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 22:01:34,907 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 22:01:35,056 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 22:01:35,057 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 22:01:35,908 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 22:01:39,221 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:01:40,150 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:01:44,221 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:01:45,151 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:01:45,151 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:01:47,903 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 22:01:49,905 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:01:50,056 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 22:01:50,056 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 22:01:55,145 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:01:55,153 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:01:55,153 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:02:00,146 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:02:00,153 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:02:00,154 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:02:05,065 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 22:02:05,066 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 22:02:05,216 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:02:07,606 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 22:02:07,608 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 22:02:07,608 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 22:02:07,610 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 22:02:07,935 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 22:02:09,937 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 22:02:10,159 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:02:10,611 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:02:15,160 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:02:15,612 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:02:17,907 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 22:02:20,062 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 22:02:20,063 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 22:02:21,155 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:02:25,161 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:02:26,155 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:02:30,162 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:02:31,164 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:02:35,069 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 22:02:35,069 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 22:02:36,196 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:02:36,541 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 22:02:36,543 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 22:02:36,543 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 22:02:36,545 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 22:02:36,966 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 22:02:37,967 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 22:02:40,170 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:02:41,546 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:02:45,170 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:02:46,547 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:02:47,910 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 22:02:50,069 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 22:02:50,070 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 22:02:52,214 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:02:55,172 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:02:57,214 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:03:00,172 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:03:02,215 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:03:05,076 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 22:03:05,081 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 22:03:08,211 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:03:08,329 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 22:03:08,330 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 22:03:08,330 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 22:03:08,332 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 22:03:08,997 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 22:03:09,998 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 22:03:10,180 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:03:13,333 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:03:15,180 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:03:17,915 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 22:03:18,917 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:03:20,075 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 22:03:20,076 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 22:03:24,230 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:03:25,274 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:03:29,230 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:03:30,274 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:03:34,237 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:03:35,082 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 22:03:35,083 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 22:03:38,896 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 22:03:38,898 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 22:03:38,898 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 22:03:38,900 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 22:03:39,030 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 22:03:39,901 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:03:40,031 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 22:03:40,303 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:03:44,902 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:03:45,304 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:03:47,918 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 22:03:49,920 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:03:50,079 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 22:03:50,080 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 22:03:55,168 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:03:55,388 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:03:55,388 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:04:00,169 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:04:00,389 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:04:00,389 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:04:05,087 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 22:04:05,088 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 22:04:05,251 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:04:08,887 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 22:04:08,889 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 22:04:08,890 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 22:04:08,891 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 22:04:09,055 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 22:04:10,056 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 22:04:10,396 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:04:10,892 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:04:15,401 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:04:15,893 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:04:17,922 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 22:04:20,082 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 22:04:20,083 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 22:04:21,216 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:04:25,403 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:04:26,216 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:04:30,404 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:04:30,404 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:04:31,217 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:04:35,096 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 22:04:35,097 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 22:04:37,181 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:04:38,843 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 22:04:38,844 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 22:04:38,845 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 22:04:38,846 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 22:04:39,080 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 22:04:40,081 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 22:04:40,408 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:04:42,847 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:04:45,409 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:04:47,848 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:04:47,926 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 22:04:50,085 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 22:04:50,086 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 22:04:53,164 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:04:55,411 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:04:58,165 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:05:00,411 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:05:03,172 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:05:05,094 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 22:05:05,097 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 22:05:08,196 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:05:08,612 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 22:05:08,613 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 22:05:08,613 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 22:05:08,614 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 22:05:09,116 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 22:05:10,117 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 22:05:10,419 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:05:13,615 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:05:15,420 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:05:17,930 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 22:05:18,932 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:05:20,184 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 22:05:20,185 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 22:05:24,349 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:05:25,421 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:05:29,349 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:05:30,431 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:05:34,360 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:05:35,104 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 22:05:35,105 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 22:05:35,918 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 22:05:35,920 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 22:05:35,921 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 22:05:35,923 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 22:05:36,139 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 22:05:38,140 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 22:05:39,924 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:05:40,428 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:05:44,925 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:05:45,429 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:05:47,934 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 22:05:49,936 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:05:50,101 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 22:05:50,102 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 22:05:55,193 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:05:55,431 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:06:00,194 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:06:00,431 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:06:05,103 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 22:06:05,103 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 22:06:05,225 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:06:07,100 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 22:06:07,102 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 22:06:07,102 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 22:06:07,104 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 22:06:07,162 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 22:06:08,164 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 22:06:10,503 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:06:10,504 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:06:11,105 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:06:15,505 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:06:15,505 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:06:16,106 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:06:17,939 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 22:06:20,104 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 22:06:20,105 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 22:06:21,259 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:06:25,507 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:06:25,507 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:06:26,260 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:06:30,517 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:06:31,266 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:06:35,103 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 22:06:35,104 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 22:06:36,035 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 22:06:36,036 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 22:06:36,037 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 22:06:36,038 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 22:06:36,186 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 22:06:37,038 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:06:38,188 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 22:06:40,561 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:06:42,039 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:06:45,562 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:06:45,562 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:06:47,039 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:06:47,942 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 22:06:50,104 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 22:06:50,104 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 22:06:52,234 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:06:55,567 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:06:55,568 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:06:57,234 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:07:00,568 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:07:00,568 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:07:02,235 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:07:05,109 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 22:07:05,114 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 22:07:07,254 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:07:08,197 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 22:07:08,199 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 22:07:08,199 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 22:07:08,200 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 22:07:08,223 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 22:07:10,225 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 22:07:10,607 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:07:10,607 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:07:13,201 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:07:15,608 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:07:17,946 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 22:07:18,948 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:07:20,106 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 22:07:20,106 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 22:07:24,198 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:07:25,609 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:07:29,199 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:07:30,610 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:07:34,200 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:07:35,113 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 22:07:35,115 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 22:07:39,243 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:07:39,911 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 22:07:39,912 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 22:07:39,912 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 22:07:39,913 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 22:07:40,259 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 22:07:40,617 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:07:42,261 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 22:07:44,914 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:07:45,618 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:07:47,950 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 22:07:49,952 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:07:50,110 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 22:07:50,111 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 22:07:55,225 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:07:55,619 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:08:00,226 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:08:00,620 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:08:05,114 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 22:08:05,120 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 22:08:05,291 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:08:09,734 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 22:08:09,735 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 22:08:09,736 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 22:08:09,738 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 22:08:10,294 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 22:08:10,629 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:08:10,739 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:08:12,296 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 22:08:15,629 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:08:15,739 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:08:17,954 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 22:08:20,114 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 22:08:20,115 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 22:08:21,195 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:08:25,630 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:08:26,196 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:08:30,631 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:08:31,197 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:08:35,114 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 22:08:35,115 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 22:08:36,215 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:08:40,639 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:08:41,218 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:08:41,267 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 22:08:41,268 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 22:08:41,268 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 22:08:41,270 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 22:08:41,318 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 22:08:42,319 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 22:08:46,271 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:08:46,339 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:08:47,959 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 22:08:50,114 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 22:08:50,115 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 22:08:52,201 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:08:55,340 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:08:55,340 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:08:57,201 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:09:00,341 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:09:00,341 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:09:02,202 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:09:05,122 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 22:09:05,131 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 22:09:07,269 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:09:10,348 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:09:10,761 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 22:09:10,762 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 22:09:10,762 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 22:09:10,764 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 22:09:11,349 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 22:09:12,350 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 22:09:12,765 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:09:15,924 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:09:15,925 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:09:17,765 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:09:17,962 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 22:09:20,117 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 22:09:20,118 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 22:09:23,240 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:09:25,926 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:09:25,926 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:09:28,240 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:09:30,927 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:09:30,927 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:09:33,241 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:09:35,117 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 22:09:35,118 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 22:09:39,237 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:09:40,933 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:09:41,068 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 22:09:41,069 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 22:09:41,069 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 22:09:41,071 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 22:09:41,376 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 22:09:42,377 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 22:09:45,072 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:09:45,967 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:09:45,968 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:09:47,966 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 22:09:50,117 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 22:09:50,118 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 22:09:50,265 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:09:55,265 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:09:55,969 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:09:55,969 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:10:00,266 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:10:00,970 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:10:00,970 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:10:05,125 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 22:10:05,131 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 22:10:05,269 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:10:10,273 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:10:10,778 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 22:10:10,780 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 22:10:10,781 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 22:10:10,782 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 22:10:11,404 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 22:10:11,932 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:10:11,933 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:10:12,407 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 22:10:15,784 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:10:16,934 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:10:17,970 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 22:10:20,119 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 22:10:20,120 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 22:10:21,280 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:10:25,941 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:10:25,942 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:10:26,281 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:10:30,942 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:10:31,282 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:10:35,119 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 22:10:35,120 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 22:10:36,290 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:10:40,947 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:10:41,288 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:10:41,818 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 22:10:41,819 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 22:10:41,820 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 22:10:41,822 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 22:10:42,439 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 22:10:44,442 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 22:10:46,096 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:10:46,096 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:10:46,823 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:10:47,974 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 22:10:50,119 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 22:10:50,120 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 22:10:52,274 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:10:55,173 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:10:55,174 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:10:57,275 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:11:00,174 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:11:02,275 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:11:05,119 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 22:11:05,120 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 22:11:07,299 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:11:10,189 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:11:12,304 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:11:12,724 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 22:11:12,726 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 22:11:12,726 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 22:11:12,727 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 22:11:13,466 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 22:11:14,467 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 22:11:15,179 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:11:17,731 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:11:17,979 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 22:11:20,119 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 22:11:20,120 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 22:11:23,288 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:11:25,180 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:11:28,289 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:11:30,181 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:11:33,289 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:11:35,120 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 22:11:35,120 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 22:11:39,243 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:11:40,186 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:11:43,260 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 22:11:43,263 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 22:11:43,263 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 22:11:43,265 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 22:11:43,496 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 22:11:44,266 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:11:44,497 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 22:11:45,187 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:11:47,982 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 22:11:49,985 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:11:50,120 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 22:11:50,120 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 22:11:55,188 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:11:55,208 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:12:00,209 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:12:00,260 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:12:05,120 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 22:12:05,120 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 22:12:05,245 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:12:10,245 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:12:10,261 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:12:15,255 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:12:15,275 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:12:17,525 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 22:12:17,527 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 22:12:17,527 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 22:12:17,529 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 22:12:17,986 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 22:12:18,529 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 22:12:18,530 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 22:12:20,120 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 22:12:20,121 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 22:12:21,239 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:12:25,273 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:12:26,239 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:12:30,274 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:12:31,240 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:12:35,120 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 22:12:35,121 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 22:12:36,247 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:12:40,276 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:12:41,248 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:12:45,283 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:12:46,255 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:12:48,001 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 22:12:48,748 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 22:12:48,750 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 22:12:48,750 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 22:12:48,752 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 22:12:49,560 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 22:12:50,120 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 22:12:50,121 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 22:12:50,561 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 22:12:52,222 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:12:55,280 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:12:57,223 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:13:00,281 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:13:02,224 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:13:05,120 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 22:13:05,121 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 22:13:07,236 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:13:10,282 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:13:12,237 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:13:15,291 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:13:17,242 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:13:18,008 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 22:13:19,864 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 22:13:19,865 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 22:13:19,866 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 22:13:19,868 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 22:13:20,307 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:13:20,312 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 22:13:20,312 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 22:13:20,589 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 22:13:20,589 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 22:13:22,476 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:13:25,382 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:13:27,476 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:13:30,383 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:13:32,477 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:13:35,307 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 22:13:35,308 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 22:13:38,427 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:13:40,439 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:13:43,433 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:13:45,452 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:13:48,007 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 22:13:48,857 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 22:13:48,858 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 22:13:48,858 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 22:13:48,858 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:13:48,861 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 22:13:49,613 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 22:13:50,307 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 22:13:50,308 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 22:13:50,613 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 22:13:54,406 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:13:55,444 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:13:59,406 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:14:00,445 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:14:04,407 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:14:05,307 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 22:14:05,308 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 22:14:09,473 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:14:10,446 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:14:14,473 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:14:15,447 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:14:18,007 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 22:14:20,017 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:14:20,316 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 22:14:20,317 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 22:14:22,555 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 22:14:22,556 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 22:14:22,557 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 22:14:22,562 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 22:14:22,640 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 22:14:24,642 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 22:14:25,452 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:14:25,563 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:14:30,453 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:14:30,564 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:14:35,311 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 22:14:35,312 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 22:14:36,482 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:14:40,454 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:14:41,483 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:14:45,463 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:14:46,488 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:14:48,016 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 22:14:50,244 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 22:14:50,245 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 22:14:50,245 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 22:14:50,247 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 22:14:50,647 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:14:50,647 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:14:50,648 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 22:14:50,649 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 22:14:50,665 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 22:14:50,666 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 22:14:51,810 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:14:55,650 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:14:55,651 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:14:56,810 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:15:00,651 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:15:00,651 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:15:01,811 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:15:05,390 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 22:15:05,390 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 22:15:07,491 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:15:10,653 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:15:10,653 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:15:12,492 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:15:15,654 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:15:15,654 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:15:17,498 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:15:18,016 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 22:15:20,394 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 22:15:20,395 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 22:15:22,905 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 22:15:22,906 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 22:15:22,907 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 22:15:22,907 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:15:22,909 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 22:15:23,697 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 22:15:24,698 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 22:15:25,661 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:15:27,910 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:15:30,662 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:15:32,910 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:15:35,585 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 22:15:35,586 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 22:15:38,652 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:15:40,663 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:15:43,652 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:15:45,664 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:15:48,029 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 22:15:49,030 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:15:50,505 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 22:15:50,506 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 22:15:53,636 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 22:15:53,638 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 22:15:53,638 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 22:15:53,640 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 22:15:53,727 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 22:15:54,642 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:15:54,728 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 22:15:56,034 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:15:56,034 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:15:59,642 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:16:01,035 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:16:01,035 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:16:04,643 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:16:05,500 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 22:16:05,500 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 22:16:10,623 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:16:11,037 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:16:11,037 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:16:15,624 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:16:16,037 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:16:16,038 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:16:18,033 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 22:16:20,508 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 22:16:20,509 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 22:16:21,614 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:16:23,573 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 22:16:23,575 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 22:16:23,576 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 22:16:23,577 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 22:16:23,751 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 22:16:25,753 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 22:16:26,043 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:16:27,578 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:16:31,044 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:16:32,579 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:16:35,501 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 22:16:35,502 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 22:16:37,628 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:16:41,045 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:16:42,628 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:16:46,046 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:16:47,629 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:16:48,028 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 22:16:50,507 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 22:16:50,514 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 22:16:52,673 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:16:54,263 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 22:16:54,265 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 22:16:54,266 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 22:16:54,271 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 22:16:54,778 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 22:16:55,779 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 22:16:56,055 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:16:58,272 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:17:01,056 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:17:03,273 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:17:05,504 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 22:17:05,505 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 22:17:08,590 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:17:11,057 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:17:13,591 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:17:16,058 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:17:16,058 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:17:18,044 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 22:17:19,046 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:17:20,511 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 22:17:20,512 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 22:17:22,873 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 22:17:22,875 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 22:17:22,875 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 22:17:22,877 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 22:17:23,809 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 22:17:24,883 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:17:25,810 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 22:17:26,274 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:17:26,274 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:17:29,880 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:17:31,275 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:17:31,275 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:17:34,880 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:17:35,508 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 22:17:35,508 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 22:17:40,618 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:17:41,460 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:17:41,460 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:17:45,618 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:17:46,461 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:17:48,038 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 22:17:50,517 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 22:17:50,521 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 22:17:50,626 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:17:55,634 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:17:55,698 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 22:17:55,700 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 22:17:55,701 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 22:17:55,704 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 22:17:55,836 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 22:17:56,468 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:17:57,837 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 22:18:00,705 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:18:01,469 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:18:05,512 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 22:18:05,513 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 22:18:06,611 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:18:11,489 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:18:11,612 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:18:16,490 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:18:16,490 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:18:16,613 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:18:18,042 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 22:18:20,516 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 22:18:20,520 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 22:18:21,681 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:18:24,755 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 22:18:24,757 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 22:18:24,757 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 22:18:24,759 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 22:18:24,863 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 22:18:25,863 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 22:18:26,502 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:18:26,502 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:18:26,760 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:18:31,503 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:18:31,503 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:18:31,760 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:18:35,513 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 22:18:35,514 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 22:18:37,626 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:18:41,504 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:18:41,505 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:18:42,626 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:18:46,516 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:18:47,630 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:18:48,052 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 22:18:50,526 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 22:18:50,529 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 22:18:51,694 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 22:18:51,696 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 22:18:51,696 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 22:18:51,697 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 22:18:51,888 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 22:18:52,698 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:18:53,889 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 22:18:56,519 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:18:56,519 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:18:57,698 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:19:01,520 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:19:01,520 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:19:02,699 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:19:05,523 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 22:19:05,524 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 22:19:08,689 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:19:11,522 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:19:13,689 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:19:16,528 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:19:18,048 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 22:19:19,050 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:19:20,523 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 22:19:20,524 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 22:19:20,552 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 22:19:20,610 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 22:19:20,611 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 22:19:20,616 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 22:19:20,914 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 22:19:21,914 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 22:19:24,618 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:19:25,527 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:19:29,618 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:19:30,528 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:19:34,619 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:19:35,523 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 22:19:35,524 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 22:19:40,529 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:19:40,594 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:19:45,536 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:19:45,601 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:19:48,055 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 22:19:48,863 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 22:19:48,865 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 22:19:48,865 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 22:19:48,867 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 22:19:48,951 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 22:19:49,952 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 22:19:50,524 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 22:19:50,525 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 22:19:50,688 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:19:55,536 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:19:55,688 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:20:00,536 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:20:00,689 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:20:05,524 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 22:20:05,524 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 22:20:05,693 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:20:10,538 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:20:10,694 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:20:15,548 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:20:15,704 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:20:16,599 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 22:20:16,601 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 22:20:16,601 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 22:20:16,605 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 22:20:16,991 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 22:20:17,992 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 22:20:18,058 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 22:20:20,524 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 22:20:20,524 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 22:20:21,690 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:20:25,546 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:20:26,690 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:20:30,547 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:20:31,691 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:20:35,524 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 22:20:35,524 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 22:20:37,659 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:20:40,559 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:20:42,666 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:20:44,418 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 22:20:44,419 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 22:20:44,419 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 22:20:44,420 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 22:20:45,019 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 22:20:45,564 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:20:46,019 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 22:20:48,062 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 22:20:48,069 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:20:50,524 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 22:20:50,524 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 22:20:53,701 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:20:55,566 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:20:58,702 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:21:00,580 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:21:03,703 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:21:05,524 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 22:21:05,525 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 22:21:09,670 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:21:10,582 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:21:14,677 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:21:15,593 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:21:16,556 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 22:21:16,557 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 22:21:16,558 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 22:21:16,560 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 22:21:17,049 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 22:21:18,050 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 22:21:18,066 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 22:21:20,069 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:21:20,524 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 22:21:20,525 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 22:21:25,607 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:21:25,704 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:21:30,608 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:21:30,744 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:21:35,524 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 22:21:35,525 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 22:21:35,610 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:21:40,614 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:21:40,747 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:21:45,625 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:21:45,659 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 22:21:45,661 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 22:21:45,662 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 22:21:45,664 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 22:21:45,807 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:21:46,079 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 22:21:48,070 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 22:21:48,081 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 22:21:50,524 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 22:21:50,525 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 22:21:51,618 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:21:56,099 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:21:56,099 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:21:56,618 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:22:01,100 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:22:01,100 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:22:01,619 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:22:05,525 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 22:22:05,525 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 22:22:07,608 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:22:11,108 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:22:12,611 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:22:15,560 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 22:22:15,562 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 22:22:15,562 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 22:22:15,564 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 22:22:16,104 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:22:16,107 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 22:22:18,075 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 22:22:18,081 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:22:18,108 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 22:22:20,525 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 22:22:20,525 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 22:22:23,648 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:22:26,105 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:22:28,648 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:22:31,106 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:22:33,649 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:22:35,526 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 22:22:35,526 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 22:22:39,616 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:22:41,107 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:22:44,623 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:22:46,124 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:22:48,078 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 22:22:48,229 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 22:22:48,230 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 22:22:48,230 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 22:22:48,232 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 22:22:49,136 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 22:22:50,137 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 22:22:50,233 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:22:50,527 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 22:22:50,528 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 22:22:55,648 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:22:56,120 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:23:00,651 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:23:01,121 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:23:05,527 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 22:23:05,527 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 22:23:05,664 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:23:10,665 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:23:11,122 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:23:15,675 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:23:16,132 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:23:18,082 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 22:23:18,118 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 22:23:18,119 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 22:23:18,120 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 22:23:18,121 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 22:23:18,164 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 22:23:20,166 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 22:23:20,527 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 22:23:20,528 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 22:23:21,643 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:23:26,263 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:23:26,643 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:23:31,264 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:23:31,264 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:23:31,644 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:23:35,527 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 22:23:35,527 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 22:23:37,617 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:23:41,265 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:23:42,618 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:23:46,273 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:23:47,625 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:23:48,084 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 22:23:48,827 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 22:23:48,829 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 22:23:48,829 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 22:23:48,831 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 22:23:49,194 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 22:23:50,195 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 22:23:50,527 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 22:23:50,528 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 22:23:52,637 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:23:56,272 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:23:56,273 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:23:57,638 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:24:01,273 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:24:01,273 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:24:02,638 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:24:05,527 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 22:24:05,528 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 22:24:07,698 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:24:11,275 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:24:11,275 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:24:12,698 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:24:16,282 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:24:17,704 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:24:18,088 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 22:24:20,532 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 22:24:20,537 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 22:24:21,721 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 22:24:21,723 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 22:24:21,724 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 22:24:21,725 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 22:24:22,224 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 22:24:22,726 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:24:24,225 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 22:24:26,283 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:24:27,726 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:24:31,283 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:24:32,727 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:24:35,530 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 22:24:35,531 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 22:24:38,680 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:24:41,285 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:24:43,680 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:24:46,285 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:24:48,093 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 22:24:49,099 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:24:50,535 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 22:24:50,536 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 22:24:54,666 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 22:24:54,668 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 22:24:54,668 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 22:24:54,669 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:24:54,670 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 22:24:55,262 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 22:24:56,263 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 22:24:56,293 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:24:59,672 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:25:01,293 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:25:04,672 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:25:05,535 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 22:25:05,535 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 22:25:09,694 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:25:11,377 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:25:11,377 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:25:14,695 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:25:16,378 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:25:16,378 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:25:18,097 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 22:25:20,102 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:25:20,542 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 22:25:20,546 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 22:25:25,384 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 22:25:25,385 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 22:25:25,385 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 22:25:25,385 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:25:25,386 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 22:25:26,292 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 22:25:26,293 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 22:25:26,381 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:25:30,388 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:25:31,382 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:25:35,388 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:25:35,537 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 22:25:35,537 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 22:25:40,601 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:25:40,618 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:25:45,602 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:25:45,619 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:25:48,101 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 22:25:50,547 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 22:25:50,548 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 22:25:50,697 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:25:55,408 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 22:25:55,410 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 22:25:55,410 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 22:25:55,411 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 22:25:55,810 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:25:56,323 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 22:25:56,323 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 22:25:56,412 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:26:00,811 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:26:01,413 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:26:05,543 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 22:26:05,544 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 22:26:06,611 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:26:10,813 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:26:11,612 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:26:15,814 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:26:16,612 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:26:18,105 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 22:26:20,550 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 22:26:20,551 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 22:26:21,647 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:26:25,616 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 22:26:25,617 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 22:26:25,618 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 22:26:25,620 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 22:26:26,001 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:26:26,352 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 22:26:27,621 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:26:28,354 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 22:26:31,002 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:26:32,622 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:26:35,548 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 22:26:35,549 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 22:26:37,680 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:26:41,058 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:26:42,681 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:26:46,059 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:26:47,682 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:26:48,109 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 22:26:50,553 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 22:26:50,554 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 22:26:53,676 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:26:55,606 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 22:26:55,608 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 22:26:55,608 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 22:26:55,609 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 22:26:56,065 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:26:56,381 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 22:26:56,382 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 22:26:59,610 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:27:01,066 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:27:04,611 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:27:05,551 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 22:27:05,552 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 22:27:09,678 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:27:11,067 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:27:14,679 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:27:16,068 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:27:18,113 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 22:27:20,121 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:27:20,553 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 22:27:20,558 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 22:27:25,466 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 22:27:25,467 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 22:27:25,468 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 22:27:25,468 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:27:25,470 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 22:27:26,075 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:27:26,411 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 22:27:26,412 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 22:27:30,471 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:27:31,076 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:27:35,472 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:27:35,553 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 22:27:35,553 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 22:27:40,631 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:27:41,175 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:27:45,631 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:27:46,176 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:27:48,118 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 22:27:50,553 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 22:27:50,554 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 22:27:51,627 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:27:56,182 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:27:56,632 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:27:59,957 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 22:27:59,958 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 22:27:59,958 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 22:27:59,960 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 22:28:00,440 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 22:28:01,348 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:28:01,961 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:28:02,441 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 22:28:05,553 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 22:28:05,554 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 22:28:07,663 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:28:11,349 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:28:12,664 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:28:16,350 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:28:17,665 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:28:18,121 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 22:28:20,553 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 22:28:20,554 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 22:28:23,662 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:28:26,361 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:28:28,669 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:28:29,972 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 22:28:29,974 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 22:28:29,974 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 22:28:29,976 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 22:28:30,462 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 22:28:31,381 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:28:32,463 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 22:28:33,977 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:28:35,554 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 22:28:35,554 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 22:28:39,656 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:28:41,382 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:28:44,657 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:28:46,383 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:28:48,125 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 22:28:50,127 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:28:50,554 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 22:28:50,555 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 22:28:55,662 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:28:56,394 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:29:00,670 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:29:01,388 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:29:01,482 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 22:29:01,484 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 22:29:01,484 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 22:29:01,485 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 22:29:01,488 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 22:29:02,489 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 22:29:05,554 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 22:29:05,555 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 22:29:06,646 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:29:10,630 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:29:11,647 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:29:15,631 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:29:16,647 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:29:18,129 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 22:29:20,554 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 22:29:20,554 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 22:29:21,720 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:29:25,644 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:29:26,728 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:29:30,638 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:29:30,663 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 22:29:30,664 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 22:29:30,665 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 22:29:30,666 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 22:29:31,522 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 22:29:32,523 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 22:29:32,666 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:29:35,554 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 22:29:35,555 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 22:29:38,647 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:29:40,639 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:29:43,648 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:29:45,640 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:29:48,133 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 22:29:49,135 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:29:50,554 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 22:29:50,554 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 22:29:54,685 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:29:55,641 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:29:59,690 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:30:00,649 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:30:01,592 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 22:30:01,593 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 22:30:01,593 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 22:30:01,595 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 22:30:02,548 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 22:30:02,549 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 22:30:05,554 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 22:30:05,555 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 22:30:05,740 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:30:10,740 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:30:10,997 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:30:15,740 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:30:15,998 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:30:18,138 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 22:30:20,554 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 22:30:20,555 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 22:30:21,648 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:30:26,005 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:30:26,657 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:30:30,753 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 22:30:30,754 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 22:30:30,755 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 22:30:30,756 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 22:30:31,182 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:30:31,578 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 22:30:31,757 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:30:32,579 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 22:30:35,555 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 22:30:35,555 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 22:30:37,656 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:30:41,184 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:30:42,657 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:30:46,270 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:30:47,658 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:30:48,141 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 22:30:50,555 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 22:30:50,555 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 22:30:52,722 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:30:56,272 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:30:57,723 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:31:01,278 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:31:02,708 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 22:31:02,710 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 22:31:02,710 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 22:31:02,712 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 22:31:03,602 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 22:31:03,713 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:31:04,603 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 22:31:05,555 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 22:31:05,555 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 22:31:09,696 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:31:11,279 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:31:14,697 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:31:16,359 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:31:18,145 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 22:31:20,148 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:31:20,555 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 22:31:20,555 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 22:31:25,646 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:31:26,360 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:31:30,652 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:31:31,370 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:31:32,590 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 22:31:32,591 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 22:31:32,592 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 22:31:32,594 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 22:31:32,632 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 22:31:34,634 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 22:31:35,555 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 22:31:35,556 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 22:31:35,716 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:31:40,716 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:31:41,369 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:31:45,717 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:31:46,370 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:31:48,149 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 22:31:50,555 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 22:31:50,556 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 22:31:51,654 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:31:56,372 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:31:56,655 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:32:01,381 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:32:01,665 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:32:04,576 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 22:32:04,577 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 22:32:04,577 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 22:32:04,579 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 22:32:04,658 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 22:32:05,555 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 22:32:05,556 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 22:32:06,660 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 22:32:06,674 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:32:11,381 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:32:11,675 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:32:16,382 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:32:16,675 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:32:18,153 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 22:32:20,555 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 22:32:20,555 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 22:32:22,649 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:32:26,383 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:32:27,649 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:32:30,674 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 22:32:31,816 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:32:31,915 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:32:33,639 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:32:38,069 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:32:38,639 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:32:38,840 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 22:32:38,841 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 22:32:43,958 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:32:46,450 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:32:46,618 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:32:48,157 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 22:32:49,160 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:32:54,160 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:32:55,138 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:32:56,355 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 22:32:56,356 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 22:32:59,482 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:33:03,917 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:33:04,466 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:33:04,483 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:33:09,484 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:33:14,484 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:33:14,902 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:33:15,265 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:33:15,354 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 22:33:15,355 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 22:33:18,161 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 22:33:20,163 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:33:25,164 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:33:25,977 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:33:30,164 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:33:30,636 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 22:33:30,637 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 22:33:32,887 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 22:33:35,637 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:33:35,638 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:33:35,962 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:33:40,659 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:33:40,659 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:33:40,962 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:33:45,637 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 22:33:45,637 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 22:33:46,761 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:33:48,165 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 22:33:50,660 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:33:52,167 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:33:55,663 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:33:57,168 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:34:00,637 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 22:34:00,637 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 22:34:02,803 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:34:05,664 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:34:07,804 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:34:10,665 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:34:12,804 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:34:15,637 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 22:34:15,637 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 22:34:18,168 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 22:34:18,169 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:34:20,667 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:34:23,170 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:34:25,667 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:34:28,171 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:34:33,171 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:34:38,172 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:34:43,173 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:34:47,330 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:34:47,334 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:34:47,334 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 22:34:47,334 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 22:34:48,172 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 22:34:48,178 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:34:52,335 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:34:52,336 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:34:53,174 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:34:57,337 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:34:57,337 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:34:58,175 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:35:02,334 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 22:35:02,335 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 22:35:03,428 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:35:07,339 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:35:07,339 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:35:08,429 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:35:12,340 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:35:12,340 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:35:13,430 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:35:17,334 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 22:35:17,335 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 22:35:18,175 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 22:35:19,178 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:35:22,342 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:35:22,342 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:35:24,178 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:35:27,343 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:35:27,343 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:35:29,179 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:35:32,334 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 22:35:32,335 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 22:35:34,500 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:35:37,345 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:35:37,345 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:35:39,501 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:35:42,346 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:35:42,346 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:35:44,502 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:35:47,335 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 22:35:47,335 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 22:35:48,180 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 22:35:50,183 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:35:52,348 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:35:52,349 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:35:55,183 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:35:57,349 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:35:57,350 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:36:00,184 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:36:02,335 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 22:36:02,335 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 22:36:05,503 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:36:07,351 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:36:07,352 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:36:10,504 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:36:12,352 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:36:12,353 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:36:15,505 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:36:17,335 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 22:36:17,335 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 22:36:18,183 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 22:36:21,186 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:36:22,354 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:36:22,354 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:36:26,187 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:36:27,355 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:36:27,356 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:36:31,187 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:36:32,335 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 22:36:32,335 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 22:36:36,493 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:36:37,357 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:36:37,357 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:36:41,493 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:36:42,358 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:36:42,358 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:36:46,494 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:36:48,188 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 22:36:52,190 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:36:57,191 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:37:02,192 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:37:07,192 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:37:12,193 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:37:17,193 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:37:18,191 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 22:37:22,197 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:37:27,197 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:37:27,241 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:37:27,242 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:37:27,247 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 22:37:27,247 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 22:37:31,174 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 22:37:32,248 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:37:32,296 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:37:35,177 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 22:37:37,249 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:37:37,484 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:37:42,242 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 22:37:42,243 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 22:37:43,378 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:37:47,758 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:37:47,758 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:37:48,196 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 22:37:49,198 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:37:52,759 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:37:52,759 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:37:54,201 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:37:57,247 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 22:37:57,248 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 22:37:59,375 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:37:59,455 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 22:37:59,457 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 22:37:59,457 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 22:37:59,459 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 22:38:00,199 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 22:38:01,200 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 22:38:02,900 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:38:02,901 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:38:04,460 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:38:07,901 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:38:09,461 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:38:12,246 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 22:38:12,247 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 22:38:15,345 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:38:17,958 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:38:17,958 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:38:18,200 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 22:38:21,202 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:38:22,958 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:38:22,959 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:38:26,210 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:38:27,256 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 22:38:27,259 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 22:38:31,391 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 22:38:31,393 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 22:38:31,393 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 22:38:31,393 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:38:31,394 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 22:38:32,235 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 22:38:32,965 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:38:33,236 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 22:38:36,396 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:38:37,966 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:38:41,396 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:38:42,250 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 22:38:42,251 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 22:38:46,412 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:38:47,968 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:38:48,205 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 22:38:52,207 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:38:52,968 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:38:57,212 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:38:57,264 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 22:38:57,268 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 22:39:01,721 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 22:39:01,722 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 22:39:01,722 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 22:39:01,723 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 22:39:02,269 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 22:39:02,724 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:39:03,168 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:39:03,168 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:39:03,269 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 22:39:07,724 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:39:08,169 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:39:08,169 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:39:12,257 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 22:39:12,257 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 22:39:13,423 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:39:18,208 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 22:39:18,214 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:39:19,210 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:39:23,215 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:39:23,215 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:39:24,211 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:39:27,257 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 22:39:27,258 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 22:39:29,431 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:39:32,963 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 22:39:32,964 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 22:39:32,965 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 22:39:32,966 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 22:39:33,302 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 22:39:33,365 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:39:34,967 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:39:35,303 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 22:39:38,366 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:39:38,366 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:39:39,967 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:39:42,257 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 22:39:42,258 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 22:39:45,362 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:39:47,439 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:39:47,525 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:39:48,212 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 22:39:51,214 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:39:52,526 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:39:52,527 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:39:56,215 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:39:57,257 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 22:39:57,258 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 22:40:01,391 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:40:02,532 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:40:05,132 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 22:40:05,135 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 22:40:05,135 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 22:40:05,136 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 22:40:05,336 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 22:40:07,137 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:40:07,337 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 22:40:07,574 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:40:12,137 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:40:12,258 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 22:40:12,258 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 22:40:17,425 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:40:17,575 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:40:18,216 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 22:40:22,576 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:40:23,218 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:40:27,258 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 22:40:27,258 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 22:40:28,393 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:40:32,582 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:40:32,582 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:40:33,397 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:40:37,364 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 22:40:37,366 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 22:40:37,366 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 22:40:37,368 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 22:40:37,794 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:40:37,795 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:40:38,369 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 22:40:39,369 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 22:40:39,370 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:40:42,258 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 22:40:42,258 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 22:40:44,441 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:40:47,796 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:40:48,220 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 22:40:50,222 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:40:52,796 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:40:55,223 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:40:57,258 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 22:40:57,259 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 22:41:00,349 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:41:02,809 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:41:05,353 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:41:06,039 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 22:41:06,040 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 22:41:06,041 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 22:41:06,042 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 22:41:06,391 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 22:41:07,392 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 22:41:07,803 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:41:11,044 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:41:12,258 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 22:41:12,259 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 22:41:16,347 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:41:17,804 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:41:18,224 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 22:41:22,227 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:41:22,903 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:41:27,228 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:41:27,258 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 22:41:27,259 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 22:41:32,385 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:41:32,916 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:41:35,181 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 22:41:35,182 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 22:41:35,183 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 22:41:35,184 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 22:41:35,412 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 22:41:37,414 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 22:41:37,916 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:41:38,184 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:41:42,258 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 22:41:42,259 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 22:41:43,423 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:41:47,917 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:41:47,918 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:41:48,228 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 22:41:49,230 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:41:52,918 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:41:54,230 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:41:57,258 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 22:41:57,259 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 22:41:59,336 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:42:02,924 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:42:04,342 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:42:05,128 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 22:42:05,130 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 22:42:05,130 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 22:42:05,132 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 22:42:05,448 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 22:42:07,449 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 22:42:07,927 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:42:10,133 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:42:12,258 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 22:42:12,259 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 22:42:15,345 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:42:17,928 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:42:18,232 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 22:42:21,234 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:42:22,929 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:42:26,235 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:42:27,259 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 22:42:27,259 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 22:42:31,365 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:42:32,934 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:42:36,373 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:42:37,905 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 22:42:37,906 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 22:42:37,907 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 22:42:37,908 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 22:42:38,050 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:42:38,306 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:42:38,478 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 22:42:39,479 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 22:42:41,909 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:42:42,259 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 22:42:42,260 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 22:42:47,308 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:42:47,411 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:42:48,235 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 22:42:52,422 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:42:53,238 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:42:57,259 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 22:42:57,260 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 22:42:58,374 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:43:02,429 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:43:03,379 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:43:06,692 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 22:43:06,693 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 22:43:06,693 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 22:43:06,694 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 22:43:07,430 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:43:07,503 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 22:43:07,503 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 22:43:08,695 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:43:12,259 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 22:43:12,260 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 22:43:14,403 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:43:17,431 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:43:18,239 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 22:43:20,242 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:43:22,432 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:43:25,242 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:43:27,259 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 22:43:27,260 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 22:43:30,383 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:43:32,433 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:43:35,387 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:43:37,436 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 22:43:37,437 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:43:37,438 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:43:37,439 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 22:43:37,439 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 22:43:37,440 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 22:43:37,530 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 22:43:39,532 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 22:43:40,441 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:43:42,259 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 22:43:42,260 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 22:43:46,348 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:43:47,444 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:43:47,444 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:43:48,244 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 22:43:52,246 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:43:52,488 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:43:57,247 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:43:57,259 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 22:43:57,260 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 22:44:02,430 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:44:02,498 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:44:06,944 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 22:44:06,945 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 22:44:06,945 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 22:44:06,947 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 22:44:07,494 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:44:07,555 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 22:44:07,556 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 22:44:07,948 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:44:12,260 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 22:44:12,260 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 22:44:13,378 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:44:17,495 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:44:18,248 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 22:44:19,250 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:44:22,612 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:44:22,612 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:44:24,251 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:44:27,260 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 22:44:27,260 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 22:44:29,428 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:44:32,619 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:44:34,432 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:44:35,342 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 22:44:35,344 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 22:44:35,344 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 22:44:35,346 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 22:44:35,583 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 22:44:37,584 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 22:44:37,772 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:44:40,347 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:44:42,260 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 22:44:42,260 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 22:44:45,358 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:44:47,774 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:44:47,774 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:44:48,250 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 22:44:51,252 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:44:52,775 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:44:52,775 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:44:56,253 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:44:57,260 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 22:44:57,260 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 22:45:01,410 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:45:02,784 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:45:05,925 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 22:45:05,926 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 22:45:05,927 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 22:45:05,928 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 22:45:06,605 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 22:45:06,928 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:45:07,606 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 22:45:07,782 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:45:11,929 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:45:12,260 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 22:45:12,260 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 22:45:17,431 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:45:17,807 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:45:17,807 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:45:18,254 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 22:45:22,808 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:45:23,257 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:45:27,263 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 22:45:27,264 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 22:45:28,357 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:45:32,432 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 22:45:32,434 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 22:45:32,434 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 22:45:32,435 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 22:45:32,633 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 22:45:32,870 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:45:32,871 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:45:33,436 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:45:33,634 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 22:45:37,871 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:45:38,437 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:45:42,262 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 22:45:42,263 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 22:45:44,374 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:45:47,873 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:45:48,258 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 22:45:50,260 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:45:52,874 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:45:52,874 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:45:55,264 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:45:57,270 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 22:45:57,271 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 22:46:00,432 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:46:00,749 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 22:46:00,750 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 22:46:00,751 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 22:46:00,752 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 22:46:01,657 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 22:46:01,658 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 22:46:03,150 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:46:03,151 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:46:05,753 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:46:08,151 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:46:08,151 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:46:10,753 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:46:12,265 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 22:46:12,265 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 22:46:16,375 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:46:18,153 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:46:18,153 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:46:18,262 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 22:46:22,264 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:46:23,154 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:46:23,154 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:46:27,273 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 22:46:27,273 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:46:27,275 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 22:46:30,052 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 22:46:30,054 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 22:46:30,054 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 22:46:30,056 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 22:46:30,683 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 22:46:31,684 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 22:46:33,057 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:46:33,198 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:46:33,198 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:46:38,058 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:46:38,199 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:46:42,267 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 22:46:42,268 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 22:46:43,337 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:46:48,200 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:46:48,267 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 22:46:49,269 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:46:53,206 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:46:54,276 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:46:57,269 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 22:46:57,269 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 22:46:57,876 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 22:46:57,877 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 22:46:57,877 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 22:46:57,879 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 22:46:58,719 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 22:46:59,720 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 22:46:59,879 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:47:02,281 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:47:02,281 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:47:04,880 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:47:07,281 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:47:07,282 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:47:09,880 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:47:12,269 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 22:47:12,270 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 22:47:15,440 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:47:17,283 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:47:17,283 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:47:18,270 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 22:47:21,272 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:47:22,284 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:47:26,275 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:47:27,277 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 22:47:27,278 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 22:47:30,523 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 22:47:30,524 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 22:47:30,525 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 22:47:30,527 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 22:47:30,749 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 22:47:31,528 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:47:31,750 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 22:47:32,293 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:47:32,294 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:47:36,528 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:47:37,294 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:47:41,529 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:47:42,271 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 22:47:42,271 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 22:47:47,296 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:47:47,296 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:47:47,406 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:47:48,274 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 22:47:52,297 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:47:52,297 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:47:53,276 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:47:57,279 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 22:47:57,282 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 22:47:58,466 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:48:02,303 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:48:02,618 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 22:48:02,619 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 22:48:02,620 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 22:48:02,622 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 22:48:02,787 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 22:48:03,623 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:48:03,788 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 22:48:07,304 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:48:08,623 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:48:12,275 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 22:48:12,276 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 22:48:14,370 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:48:17,306 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:48:18,278 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 22:48:20,280 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:48:22,307 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:48:25,281 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:48:27,275 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 22:48:27,276 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 22:48:30,387 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:48:32,312 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:48:35,222 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 22:48:35,224 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 22:48:35,225 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 22:48:35,226 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 22:48:35,824 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 22:48:35,825 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 22:48:36,227 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:48:37,622 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:48:37,622 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:48:41,228 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:48:42,275 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 22:48:42,276 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 22:48:46,374 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:48:47,623 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:48:47,624 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:48:48,282 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 22:48:52,284 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:48:52,652 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:48:52,652 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:48:57,275 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 22:48:57,276 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 22:48:57,404 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:49:02,405 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:49:02,662 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:49:07,408 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:49:07,590 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 22:49:07,592 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 22:49:07,593 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 22:49:07,594 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 22:49:07,736 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:49:07,860 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 22:49:07,998 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:49:10,862 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 22:49:12,275 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 22:49:12,276 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 22:49:12,419 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:49:17,420 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:49:18,000 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:49:18,286 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 22:49:23,001 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:49:23,001 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:49:23,288 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:49:27,276 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 22:49:27,276 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 22:49:28,447 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:49:33,002 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:49:33,448 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:49:38,015 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:49:38,463 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:49:39,218 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 22:49:39,220 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 22:49:39,221 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 22:49:39,222 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 22:49:39,887 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 22:49:40,888 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 22:49:42,276 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 22:49:42,276 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 22:49:44,403 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:49:48,013 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:49:48,014 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:49:48,291 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 22:49:50,293 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:49:53,014 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:49:55,294 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:49:57,276 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 22:49:57,276 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 22:50:00,416 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:50:03,015 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:50:05,417 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:50:08,021 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:50:10,424 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:50:11,747 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 22:50:11,748 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 22:50:11,748 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 22:50:11,750 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 22:50:11,928 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 22:50:12,869 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 22:50:12,870 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 22:50:14,930 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 22:50:16,027 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:50:18,294 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 22:50:18,868 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:50:21,296 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:50:23,869 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:50:26,297 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:50:27,868 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 22:50:27,869 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 22:50:32,039 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:50:32,870 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:50:32,871 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:50:37,042 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:50:37,885 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:50:41,869 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 22:50:41,871 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 22:50:41,872 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 22:50:41,873 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 22:50:41,969 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 22:50:42,874 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:50:42,923 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 22:50:42,924 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 22:50:44,971 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 22:50:48,020 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:50:48,021 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:50:48,030 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:50:48,298 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 22:50:53,021 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:50:53,300 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:50:57,922 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 22:50:57,923 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 22:50:59,046 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:51:03,022 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:51:03,023 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:51:04,047 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:51:08,027 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:51:09,052 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:51:12,925 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 22:51:12,925 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 22:51:13,741 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 22:51:13,743 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 22:51:13,744 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 22:51:13,746 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 22:51:13,992 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 22:51:14,747 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:51:16,994 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 22:51:18,175 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:51:18,175 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:51:18,302 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 22:51:20,304 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:51:23,176 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:51:23,176 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:51:25,305 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:51:27,925 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 22:51:27,926 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 22:51:31,086 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:51:33,177 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:51:33,177 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:51:36,086 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:51:38,190 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:51:41,091 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:51:42,928 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 22:51:42,928 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 22:51:43,485 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 22:51:43,487 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 22:51:43,487 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 22:51:43,489 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 22:51:44,020 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 22:51:45,021 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 22:51:46,490 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:51:48,183 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:51:48,305 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 22:51:52,307 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:51:53,184 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:51:57,308 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:51:57,928 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 22:51:57,929 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 22:52:03,035 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:52:03,185 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:52:08,038 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:52:08,193 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:52:11,498 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 22:52:11,500 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 22:52:11,500 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 22:52:11,501 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 22:52:12,049 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 22:52:12,928 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 22:52:12,929 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 22:52:13,050 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 22:52:14,027 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:52:18,310 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 22:52:18,505 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:52:19,312 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:52:23,506 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:52:23,506 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:52:24,313 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:52:27,928 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 22:52:27,929 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 22:52:30,090 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:52:33,508 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:52:33,508 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:52:35,091 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:52:38,510 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:52:40,097 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:52:42,692 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 22:52:42,694 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 22:52:42,694 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 22:52:42,695 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 22:52:43,085 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 22:52:43,096 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 22:52:43,097 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 22:52:45,086 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 22:52:45,245 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:52:48,312 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 22:52:48,525 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:52:48,525 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:52:50,314 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:52:53,526 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:52:53,526 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:52:55,314 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:52:58,096 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 22:52:58,097 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 22:53:01,248 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:53:03,527 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:53:03,528 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:53:06,260 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:53:08,532 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:53:11,257 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:53:11,589 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 22:53:11,590 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 22:53:11,590 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 22:53:11,591 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 22:53:12,116 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 22:53:13,096 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 22:53:13,097 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 22:53:13,117 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 22:53:17,189 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:53:18,316 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 22:53:18,533 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:53:22,318 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:53:23,534 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:53:27,319 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:53:28,096 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 22:53:28,096 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 22:53:33,271 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:53:33,535 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:53:38,278 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:53:38,544 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:53:40,698 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 22:53:40,699 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 22:53:40,700 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 22:53:40,701 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 22:53:41,141 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 22:53:43,096 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 22:53:43,097 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 22:53:43,142 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 22:53:44,168 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:53:48,318 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 22:53:48,829 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:53:49,321 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:53:53,829 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:53:54,321 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:53:58,096 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 22:53:58,097 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 22:54:00,280 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:54:03,831 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:54:05,291 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:54:08,840 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:54:10,294 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:54:10,728 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 22:54:10,729 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 22:54:10,729 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 22:54:10,731 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 22:54:11,166 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 22:54:13,096 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 22:54:13,097 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 22:54:13,167 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 22:54:16,222 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:54:18,322 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 22:54:18,874 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:54:18,875 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:54:21,325 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:54:23,875 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:54:23,876 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:54:26,326 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:54:28,096 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 22:54:28,097 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 22:54:32,226 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:54:33,877 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:54:33,877 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:54:37,234 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:54:38,881 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:54:41,434 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 22:54:41,436 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 22:54:41,436 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 22:54:41,438 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 22:54:42,195 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 22:54:42,439 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:54:43,096 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 22:54:43,097 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 22:54:43,196 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 22:54:48,227 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:54:48,326 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 22:54:48,632 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:54:53,329 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:54:53,632 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:54:58,097 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 22:54:58,097 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 22:54:59,181 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:55:03,697 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:55:04,182 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:55:08,705 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:55:09,190 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:55:12,407 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 22:55:12,409 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 22:55:12,410 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 22:55:12,411 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 22:55:13,097 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 22:55:13,098 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 22:55:13,223 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 22:55:13,223 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 22:55:14,232 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:55:18,330 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 22:55:18,918 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:55:18,918 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:55:19,333 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:55:23,919 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:55:23,919 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:55:24,333 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:55:28,097 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 22:55:28,098 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 22:55:30,233 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:55:33,920 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:55:33,921 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:55:35,233 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:55:38,927 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:55:38,929 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:55:40,239 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:55:41,699 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 22:55:41,700 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 22:55:41,700 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 22:55:41,701 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 22:55:42,245 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 22:55:43,097 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 22:55:43,098 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 22:55:43,246 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 22:55:45,258 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:55:48,335 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 22:55:48,608 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:55:48,609 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:55:50,337 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:55:53,609 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:55:53,610 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:55:55,337 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:55:58,097 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 22:55:58,098 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 22:56:01,215 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:56:03,611 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:56:03,612 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:56:06,215 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:56:08,622 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:56:11,222 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:56:13,097 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 22:56:13,097 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 22:56:13,973 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 22:56:13,974 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 22:56:13,974 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 22:56:13,975 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 22:56:14,274 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 22:56:15,276 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 22:56:16,976 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:56:18,339 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 22:56:18,628 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:56:18,628 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:56:22,341 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:56:23,629 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:56:23,629 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:56:27,342 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:56:28,097 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 22:56:28,098 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 22:56:33,254 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:56:33,631 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:56:33,631 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:56:38,265 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:56:38,635 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:56:43,097 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 22:56:43,098 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 22:56:43,623 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 22:56:43,624 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 22:56:43,624 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 22:56:43,625 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:56:43,625 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 22:56:44,297 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 22:56:45,299 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 22:56:48,343 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 22:56:49,027 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:56:49,345 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:56:54,027 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:56:54,027 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:56:54,345 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:56:58,098 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 22:56:58,098 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 22:57:00,179 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:57:04,029 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:57:05,179 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:57:09,040 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:57:10,187 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:57:13,100 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 22:57:13,101 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 22:57:14,200 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 22:57:14,201 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 22:57:14,202 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 22:57:14,203 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 22:57:14,326 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 22:57:15,204 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:57:15,327 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 22:57:18,347 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 22:57:19,035 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:57:20,349 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:57:24,036 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:57:25,349 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:57:28,099 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 22:57:28,100 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 22:57:31,173 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:57:34,071 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:57:34,071 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:57:36,174 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:57:39,072 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:57:39,072 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:57:41,179 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:57:43,103 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 22:57:43,104 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 22:57:46,217 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:57:46,634 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 22:57:46,635 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 22:57:46,636 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 22:57:46,637 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 22:57:47,356 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 22:57:48,351 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 22:57:49,076 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:57:49,360 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 22:57:51,354 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:57:54,076 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:57:56,354 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:57:58,102 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 22:57:58,103 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 22:58:02,220 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:58:03,211 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:58:03,211 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:58:07,221 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:58:08,212 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:58:08,212 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:58:12,221 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:58:13,102 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 22:58:13,103 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 22:58:18,214 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:58:18,216 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:58:18,358 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 22:58:19,112 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 22:58:19,114 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 22:58:19,114 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 22:58:19,116 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 22:58:19,385 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 22:58:21,386 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 22:58:23,260 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:58:23,261 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:58:24,117 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:58:28,102 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 22:58:28,103 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 22:58:29,221 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:58:33,262 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:58:34,221 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:58:38,262 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:58:38,263 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:58:39,222 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:58:43,102 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 22:58:43,103 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 22:58:44,277 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:58:47,776 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 22:58:47,778 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 22:58:47,778 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 22:58:47,779 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 22:58:48,270 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:58:48,362 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 22:58:48,409 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 22:58:49,364 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:58:49,410 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 22:58:53,270 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:58:54,365 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:58:58,103 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 22:58:58,103 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 22:59:00,268 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:59:03,375 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:59:05,268 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:59:08,376 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:59:10,269 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:59:13,103 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 22:59:13,103 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 22:59:16,212 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:59:17,838 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 22:59:17,839 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 22:59:17,840 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 22:59:17,841 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 22:59:18,366 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 22:59:18,386 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:59:18,434 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 22:59:19,435 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 22:59:21,369 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:59:23,387 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:59:26,369 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:59:28,103 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 22:59:28,104 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 22:59:32,238 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:59:33,388 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:59:37,238 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:59:38,389 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:59:42,239 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:59:43,103 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 22:59:43,104 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 22:59:48,217 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:59:48,378 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 22:59:48,399 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:59:49,986 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 22:59:49,988 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 22:59:49,988 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 22:59:49,990 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 22:59:50,459 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 22:59:51,459 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 22:59:53,399 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 22:59:53,991 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 22:59:58,103 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 22:59:58,104 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 22:59:59,273 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:00:03,400 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:00:04,273 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:00:08,401 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:00:09,274 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:00:13,103 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 23:00:13,104 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 23:00:14,832 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:00:18,379 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 23:00:18,408 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:00:19,280 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 23:00:19,282 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 23:00:19,282 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 23:00:19,285 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 23:00:19,485 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 23:00:20,286 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:00:21,487 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 23:00:23,434 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:00:25,286 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:00:28,103 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 23:00:28,104 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 23:00:31,204 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:00:33,436 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:00:36,205 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:00:38,437 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:00:41,206 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:00:43,103 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 23:00:43,104 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 23:00:46,221 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:00:48,380 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 23:00:48,446 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:00:49,159 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 23:00:49,161 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 23:00:49,162 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 23:00:49,164 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 23:00:49,517 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 23:00:51,519 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 23:00:52,165 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:00:53,561 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:00:57,166 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:00:58,103 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 23:00:58,104 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 23:01:02,184 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:01:03,562 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:01:07,185 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:01:08,563 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:01:12,185 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:01:13,104 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 23:01:13,104 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 23:01:17,261 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:01:18,389 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 23:01:18,574 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:01:19,434 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 23:01:19,435 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 23:01:19,436 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 23:01:19,438 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 23:01:19,535 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 23:01:21,537 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 23:01:22,439 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:01:23,580 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:01:27,439 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:01:28,104 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 23:01:28,104 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 23:01:33,198 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:01:33,581 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:01:38,198 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:01:38,581 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:01:43,104 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 23:01:43,105 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 23:01:43,215 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:01:48,220 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:01:48,389 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 23:01:48,591 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:01:50,137 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 23:01:50,139 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 23:01:50,139 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 23:01:50,141 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 23:01:50,561 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 23:01:51,562 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 23:01:53,590 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:01:54,142 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:01:58,104 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 23:01:58,104 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 23:01:59,290 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:02:03,598 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:02:04,290 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:02:08,598 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:02:09,291 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:02:13,104 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 23:02:13,105 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 23:02:15,178 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:02:18,391 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 23:02:18,610 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:02:18,932 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 23:02:18,933 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 23:02:18,934 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 23:02:18,937 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 23:02:19,586 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 23:02:19,587 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 23:02:20,938 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:02:23,609 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:02:25,939 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:02:28,104 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 23:02:28,105 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 23:02:31,231 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:02:33,684 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:02:36,232 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:02:38,770 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:02:41,233 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:02:43,104 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 23:02:43,105 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 23:02:46,262 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:02:48,398 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 23:02:48,781 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:02:51,402 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:02:52,955 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 23:02:52,956 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 23:02:52,957 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 23:02:52,958 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 23:02:53,611 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 23:02:53,612 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 23:02:53,777 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:02:56,959 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:02:58,104 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 23:02:58,105 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 23:03:02,229 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:03:03,780 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:03:07,230 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:03:08,876 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:03:12,230 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:03:13,105 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 23:03:13,105 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 23:03:18,105 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:03:18,230 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:03:18,399 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 23:03:23,116 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:03:23,411 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:03:24,660 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 23:03:24,662 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 23:03:24,663 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 23:03:24,664 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 23:03:25,642 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 23:03:25,642 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 23:03:28,105 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 23:03:28,106 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 23:03:29,200 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:03:33,112 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:03:34,200 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:03:38,113 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:03:39,201 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:03:43,105 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 23:03:43,105 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 23:03:44,225 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:03:48,120 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:03:48,406 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 23:03:49,406 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:03:53,120 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:03:53,814 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 23:03:53,815 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 23:03:53,816 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 23:03:53,818 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 23:03:54,671 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 23:03:54,819 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:03:55,673 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 23:03:58,105 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 23:03:58,106 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 23:04:00,175 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:04:03,213 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:04:05,176 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:04:08,244 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:04:10,177 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:04:13,105 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 23:04:13,106 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 23:04:15,299 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:04:18,251 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:04:18,419 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 23:04:20,420 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:04:23,252 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:04:23,777 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 23:04:23,778 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 23:04:23,778 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 23:04:23,779 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 23:04:24,702 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 23:04:25,704 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 23:04:25,780 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:04:28,105 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 23:04:28,106 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 23:04:31,238 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:04:33,254 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:04:33,254 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:04:36,238 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:04:38,334 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:04:41,239 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:04:43,105 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 23:04:43,106 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 23:04:46,247 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:04:48,335 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:04:48,411 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 23:04:51,417 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:04:53,345 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:04:55,329 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 23:04:55,331 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 23:04:55,331 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 23:04:55,334 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 23:04:55,731 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 23:04:57,335 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:04:57,732 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 23:04:58,105 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 23:04:58,106 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 23:05:03,259 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:05:03,476 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:05:03,477 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:05:08,260 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:05:08,477 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:05:08,478 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:05:13,106 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 23:05:13,106 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 23:05:14,197 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:05:18,412 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 23:05:18,480 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:05:18,480 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:05:19,414 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:05:23,481 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:05:23,481 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:05:24,414 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:05:28,111 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 23:05:28,116 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 23:05:30,187 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:05:30,382 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 23:05:30,384 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 23:05:30,384 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 23:05:30,389 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 23:05:30,757 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 23:05:31,758 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 23:05:33,533 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:05:35,391 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:05:38,534 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:05:38,534 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:05:40,391 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:05:43,110 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 23:05:43,111 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 23:05:46,184 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:05:48,416 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 23:05:48,536 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:05:48,536 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:05:51,419 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:05:53,537 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:05:53,537 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:05:56,423 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:05:58,113 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 23:05:58,119 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 23:06:01,520 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 23:06:01,522 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 23:06:01,523 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 23:06:01,523 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:06:01,528 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 23:06:01,788 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 23:06:03,672 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:06:03,789 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 23:06:06,529 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:06:08,673 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:06:11,530 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:06:13,113 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 23:06:13,114 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 23:06:17,272 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:06:18,421 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 23:06:18,674 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:06:22,424 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:06:23,675 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:06:23,675 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:06:27,424 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:06:28,113 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 23:06:28,114 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 23:06:33,286 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:06:33,686 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:06:35,294 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 23:06:35,295 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 23:06:35,296 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 23:06:35,298 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 23:06:35,824 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 23:06:35,825 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 23:06:38,299 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:06:38,698 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:06:38,698 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:06:43,113 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 23:06:43,114 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 23:06:44,270 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:06:48,424 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 23:06:48,926 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:06:48,927 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:06:49,427 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:06:53,927 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:06:53,928 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:06:54,427 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:06:58,113 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 23:06:58,114 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 23:07:00,290 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:07:03,944 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:07:05,297 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:07:05,529 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 23:07:05,530 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 23:07:05,531 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 23:07:05,533 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 23:07:05,855 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 23:07:07,857 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 23:07:08,938 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:07:08,938 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:07:10,534 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:07:13,113 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 23:07:13,114 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 23:07:16,230 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:07:18,428 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 23:07:19,105 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:07:19,105 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:07:21,431 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:07:24,106 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:07:24,106 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:07:26,431 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:07:28,114 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 23:07:28,114 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 23:07:32,230 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:07:34,119 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:07:35,666 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 23:07:35,666 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 23:07:35,667 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 23:07:35,670 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 23:07:35,883 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 23:07:37,671 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:07:37,884 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 23:07:39,113 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:07:39,114 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:07:42,672 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:07:43,114 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 23:07:43,114 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 23:07:48,184 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:07:48,184 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:07:48,278 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:07:48,432 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 23:07:53,184 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:07:53,435 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:07:58,121 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 23:07:58,122 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 23:07:59,220 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:08:03,181 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 23:08:03,183 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 23:08:03,183 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 23:08:03,184 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 23:08:03,328 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:08:03,586 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:08:03,923 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 23:08:03,924 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 23:08:05,185 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:08:08,586 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:08:10,186 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:08:13,118 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 23:08:13,119 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 23:08:15,272 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:08:18,436 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 23:08:18,653 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:08:18,653 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:08:20,439 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:08:23,654 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:08:23,655 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:08:25,448 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:08:28,124 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 23:08:28,125 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 23:08:29,913 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 23:08:29,915 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 23:08:29,916 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 23:08:29,917 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 23:08:29,949 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 23:08:30,918 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:08:31,951 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 23:08:33,661 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:08:35,918 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:08:38,661 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:08:40,919 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:08:43,122 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 23:08:43,123 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 23:08:46,199 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:08:48,441 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 23:08:48,793 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:08:51,443 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:08:53,794 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:08:53,794 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:08:56,453 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:08:58,130 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 23:08:58,139 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 23:09:00,301 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 23:09:00,303 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 23:09:00,303 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 23:09:00,305 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 23:09:00,977 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 23:09:01,978 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 23:09:02,306 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:09:03,803 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:09:07,306 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:09:08,804 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:09:12,307 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:09:13,127 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 23:09:13,128 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 23:09:18,254 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:09:18,444 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 23:09:18,805 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:09:23,447 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:09:23,806 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:09:28,132 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 23:09:28,133 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 23:09:29,257 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:09:30,375 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 23:09:30,376 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 23:09:30,377 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 23:09:30,378 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 23:09:31,010 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 23:09:32,011 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 23:09:33,887 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:09:33,887 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:09:34,380 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:09:38,888 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:09:38,889 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:09:39,380 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:09:43,129 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 23:09:43,130 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 23:09:45,219 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:09:48,448 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 23:09:48,890 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:09:48,890 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:09:50,451 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:09:53,890 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:09:55,451 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:09:58,137 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 23:09:58,140 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 23:10:01,217 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:10:01,386 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 23:10:01,388 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 23:10:01,388 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 23:10:01,390 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 23:10:02,037 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 23:10:02,037 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 23:10:03,899 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:10:06,392 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:10:08,900 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:10:11,392 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:10:13,344 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 23:10:13,345 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 23:10:16,512 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:10:18,453 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 23:10:18,901 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:10:22,455 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:10:23,902 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:10:27,465 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:10:28,346 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 23:10:28,350 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 23:10:30,768 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 23:10:30,769 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 23:10:30,770 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 23:10:30,771 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 23:10:31,060 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 23:10:32,061 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 23:10:32,771 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:10:33,915 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:10:37,772 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:10:38,916 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:10:42,773 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:10:43,343 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 23:10:43,344 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 23:10:48,457 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 23:10:48,463 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:10:48,917 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:10:53,459 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:10:53,918 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:10:58,352 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 23:10:58,358 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 23:10:58,476 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:11:00,000 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 23:11:00,002 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 23:11:00,002 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 23:11:00,003 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 23:11:00,086 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 23:11:02,087 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 23:11:03,405 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:11:04,004 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:11:08,406 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:11:09,005 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:11:13,346 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 23:11:13,347 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 23:11:14,461 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:11:18,462 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 23:11:18,640 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:11:19,464 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:11:23,641 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:11:24,465 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:11:28,350 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 23:11:28,354 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 23:11:30,459 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:11:31,626 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 23:11:31,628 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 23:11:31,628 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 23:11:31,629 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 23:11:32,116 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 23:11:32,116 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 23:11:33,771 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:11:35,630 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:11:38,772 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:11:40,631 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:11:43,349 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 23:11:43,350 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 23:11:46,489 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:11:48,465 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 23:11:48,773 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:11:52,470 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:11:53,789 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:11:57,477 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:11:58,337 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 23:11:58,339 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 23:11:58,340 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 23:11:58,341 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 23:11:58,707 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 23:11:58,708 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 23:11:59,139 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 23:12:00,140 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 23:12:02,877 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:12:03,785 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:12:07,878 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:12:08,786 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:12:12,878 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:12:13,483 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 23:12:13,484 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 23:12:18,468 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 23:12:18,475 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:12:18,787 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:12:23,476 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:12:23,794 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:12:27,522 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 23:12:27,524 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 23:12:27,525 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 23:12:27,526 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 23:12:28,163 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 23:12:28,483 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 23:12:28,484 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 23:12:28,592 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:12:29,164 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 23:12:33,592 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:12:33,931 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:12:38,593 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:12:38,932 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:12:43,483 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 23:12:43,484 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 23:12:43,653 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:12:48,472 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 23:12:48,933 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:12:49,483 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:12:53,941 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:12:54,478 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:12:54,946 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 23:12:54,948 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 23:12:54,948 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 23:12:54,950 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 23:12:55,192 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 23:12:57,194 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 23:12:58,484 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 23:12:58,484 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 23:12:59,659 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:13:03,942 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:13:04,659 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:13:08,947 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:13:09,660 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:13:13,484 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 23:13:13,484 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 23:13:15,609 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:13:18,476 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 23:13:18,948 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:13:21,484 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:13:23,959 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:13:26,485 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:13:26,914 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 23:13:26,915 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 23:13:26,916 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 23:13:26,917 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 23:13:27,222 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 23:13:28,484 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 23:13:28,485 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 23:13:29,224 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 23:13:31,607 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:13:34,061 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:13:36,608 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:13:39,062 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:13:41,609 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:13:43,484 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 23:13:43,484 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 23:13:47,556 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:13:48,480 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 23:13:49,063 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:13:53,496 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:13:54,074 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:13:55,937 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 23:13:55,939 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 23:13:55,939 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 23:13:55,940 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 23:13:56,255 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 23:13:57,256 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 23:13:58,484 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 23:13:58,485 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 23:13:58,646 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:14:03,648 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:14:04,091 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:14:08,648 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:14:09,091 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:14:13,484 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 23:14:13,485 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 23:14:14,551 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:14:18,485 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 23:14:19,133 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:14:20,487 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:14:24,145 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:14:25,492 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:14:28,489 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 23:14:28,493 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 23:14:28,735 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 23:14:28,736 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 23:14:28,736 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 23:14:28,738 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 23:14:29,290 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 23:14:30,739 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:14:31,292 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 23:14:34,143 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:14:35,740 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:14:39,144 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:14:40,741 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:14:43,487 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 23:14:43,488 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 23:14:46,654 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:14:48,487 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 23:14:48,489 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:14:52,494 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:14:53,497 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:14:57,504 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:14:57,936 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 23:14:57,938 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 23:14:57,938 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 23:14:57,939 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 23:14:58,319 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 23:14:58,487 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 23:14:58,488 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 23:14:59,320 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 23:15:02,658 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:15:03,498 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:15:07,659 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:15:08,499 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:15:12,659 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:15:13,487 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 23:15:13,488 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 23:15:18,491 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 23:15:18,497 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:15:18,500 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:15:23,494 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:15:23,501 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:15:28,491 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 23:15:28,498 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 23:15:28,590 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:15:32,950 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 23:15:32,952 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 23:15:32,952 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 23:15:32,954 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 23:15:33,351 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 23:15:33,508 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:15:33,955 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:15:35,353 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 23:15:38,509 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:15:38,956 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:15:43,487 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 23:15:43,488 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 23:15:44,556 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:15:48,495 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 23:15:48,510 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:15:50,498 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:15:53,511 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:15:55,498 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:15:58,488 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 23:15:58,488 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 23:16:00,572 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:16:03,520 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:16:05,376 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 23:16:05,378 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 23:16:05,378 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 23:16:05,380 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 23:16:05,381 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 23:16:06,381 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:16:07,383 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 23:16:08,524 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:16:08,524 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:16:11,381 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:16:13,488 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 23:16:13,488 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 23:16:16,665 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:16:18,500 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 23:16:18,569 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:16:18,569 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:16:22,503 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:16:23,570 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:16:27,504 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:16:28,488 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 23:16:28,488 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 23:16:32,569 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:16:33,578 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:16:35,987 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 23:16:35,989 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 23:16:35,989 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 23:16:35,991 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 23:16:36,408 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 23:16:37,409 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 23:16:37,992 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:16:38,579 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:16:38,579 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:16:42,993 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:16:43,488 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 23:16:43,489 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 23:16:48,504 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 23:16:48,510 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:16:48,773 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:16:48,773 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:16:53,507 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:16:53,774 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:16:58,488 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 23:16:58,489 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 23:16:58,564 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:17:03,574 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:17:03,783 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:17:06,924 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 23:17:06,926 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 23:17:06,927 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 23:17:06,929 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 23:17:07,437 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 23:17:08,783 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:17:08,930 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:17:09,439 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 23:17:13,488 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 23:17:13,489 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 23:17:14,567 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:17:18,507 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 23:17:18,784 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:17:20,510 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:17:23,785 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:17:25,511 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:17:28,488 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 23:17:28,489 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 23:17:30,614 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:17:33,799 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:17:35,619 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:17:35,964 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 23:17:35,965 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 23:17:35,966 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 23:17:35,967 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 23:17:36,465 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 23:17:37,466 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 23:17:38,793 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:17:40,968 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:17:43,489 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 23:17:43,489 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 23:17:46,657 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:17:48,511 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 23:17:48,795 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:17:52,514 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:17:53,795 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:17:57,515 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:17:58,489 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 23:17:58,489 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 23:18:02,641 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:18:03,803 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:18:03,803 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:18:04,327 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 23:18:04,328 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 23:18:04,329 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 23:18:04,330 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 23:18:04,488 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 23:18:05,490 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 23:18:08,332 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:18:08,806 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:18:08,806 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:18:13,332 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:18:13,489 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 23:18:13,489 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 23:18:18,515 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 23:18:18,522 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:18:18,865 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:18:18,865 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:18:23,518 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:18:23,866 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:18:23,866 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:18:28,489 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 23:18:28,489 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 23:18:28,662 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:18:33,330 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 23:18:33,331 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 23:18:33,332 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 23:18:33,333 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 23:18:33,516 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 23:18:33,872 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:18:34,334 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:18:35,517 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 23:18:38,873 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:18:39,334 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:18:43,489 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 23:18:43,489 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 23:18:44,631 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:18:48,519 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 23:18:48,874 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:18:50,522 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:18:53,875 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:18:55,522 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:18:58,498 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 23:18:58,499 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 23:19:00,660 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:19:02,093 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 23:19:02,094 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 23:19:02,094 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 23:19:02,095 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 23:19:02,540 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 23:19:03,541 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 23:19:03,881 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:19:06,096 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:19:08,882 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:19:11,097 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:19:13,493 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 23:19:13,494 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 23:19:16,607 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:19:18,524 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 23:19:18,883 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:19:22,526 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:19:23,884 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:19:27,536 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:19:28,506 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 23:19:28,506 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 23:19:32,601 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:19:33,162 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 23:19:33,163 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 23:19:33,163 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 23:19:33,164 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 23:19:33,580 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 23:19:33,891 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:19:35,581 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 23:19:38,166 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:19:38,892 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:19:43,166 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:19:43,499 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 23:19:43,500 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 23:19:48,527 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 23:19:48,534 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:19:48,893 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:19:53,530 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:19:53,894 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:19:58,507 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 23:19:58,516 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 23:19:58,675 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:20:03,676 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:20:03,902 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:20:03,967 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 23:20:03,968 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 23:20:03,969 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 23:20:03,971 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 23:20:04,603 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 23:20:05,604 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 23:20:08,903 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:20:08,972 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:20:13,502 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 23:20:13,503 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 23:20:14,648 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:20:18,533 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 23:20:18,904 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:20:20,536 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:20:23,906 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:20:25,536 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:20:28,503 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 23:20:28,504 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 23:20:30,628 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:20:33,641 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 23:20:33,642 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 23:20:33,642 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 23:20:33,644 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 23:20:34,062 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:20:34,062 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:20:34,628 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 23:20:35,629 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 23:20:35,644 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:20:39,063 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:20:40,645 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:20:43,502 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 23:20:43,503 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 23:20:45,668 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:20:48,536 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 23:20:49,064 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:20:51,538 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:20:54,065 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:20:54,065 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:20:56,539 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:20:58,518 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 23:20:58,525 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 23:21:01,658 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:21:03,869 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 23:21:03,871 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 23:21:03,872 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 23:21:03,873 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 23:21:04,279 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:21:04,279 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:21:04,654 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 23:21:05,655 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 23:21:06,874 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:21:09,280 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:21:09,280 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:21:11,875 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:21:13,508 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 23:21:13,508 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 23:21:17,674 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:21:18,540 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 23:21:19,282 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:21:19,282 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:21:23,542 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:21:24,282 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:21:24,283 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:21:28,511 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 23:21:28,512 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 23:21:28,645 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:21:32,974 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 23:21:32,976 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 23:21:32,976 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 23:21:32,978 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 23:21:33,679 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 23:21:33,979 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:21:34,377 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:21:35,681 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 23:21:38,979 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:21:39,378 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:21:39,378 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:21:43,508 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 23:21:43,509 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 23:21:44,673 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:21:48,544 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 23:21:49,380 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:21:49,380 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:21:50,546 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:21:54,381 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:21:54,381 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:21:55,547 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:21:58,508 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 23:21:58,509 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 23:22:00,623 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:22:04,398 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:22:05,330 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 23:22:05,332 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 23:22:05,333 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 23:22:05,334 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 23:22:05,707 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 23:22:06,335 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:22:07,708 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 23:22:09,482 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:22:09,483 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:22:11,336 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:22:13,509 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 23:22:13,509 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 23:22:16,598 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:22:18,549 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 23:22:19,484 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:22:19,484 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:22:22,551 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:22:24,485 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:22:24,485 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:22:27,552 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:22:28,509 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 23:22:28,509 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 23:22:32,598 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:22:34,493 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:22:34,829 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 23:22:34,831 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 23:22:34,831 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 23:22:34,833 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 23:22:35,734 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 23:22:37,735 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 23:22:37,834 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:22:39,494 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:22:42,835 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:22:43,509 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 23:22:43,509 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 23:22:48,552 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 23:22:48,558 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:22:49,495 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:22:53,555 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:22:54,496 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:22:58,516 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 23:22:58,517 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 23:22:58,647 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:23:01,492 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 23:23:01,493 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 23:23:01,494 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 23:23:01,496 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 23:23:01,757 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 23:23:03,759 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 23:23:03,892 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:23:04,497 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:23:08,893 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:23:08,893 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:23:09,497 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:23:13,512 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 23:23:13,513 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 23:23:14,679 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:23:18,556 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 23:23:19,020 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:23:20,558 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:23:24,021 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:23:25,559 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:23:28,518 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 23:23:28,519 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 23:23:30,515 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 23:23:30,517 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 23:23:30,518 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 23:23:30,519 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 23:23:30,786 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 23:23:31,520 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:23:31,787 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 23:23:34,030 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:23:36,521 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:23:39,030 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:23:41,521 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:23:43,516 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 23:23:43,517 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 23:23:46,633 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:23:48,560 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 23:23:49,086 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:23:49,086 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:23:52,562 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:23:54,087 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:23:54,087 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:23:57,567 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:23:58,518 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 23:23:58,518 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 23:24:00,466 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 23:24:00,468 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 23:24:00,468 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 23:24:00,469 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 23:24:00,814 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 23:24:01,815 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 23:24:03,470 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:24:04,091 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:24:08,471 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:24:09,092 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:24:13,471 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:24:13,518 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 23:24:13,518 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 23:24:18,564 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 23:24:18,570 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:24:19,093 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:24:23,567 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:24:24,094 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:24:28,525 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 23:24:28,532 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 23:24:28,675 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:24:30,496 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 23:24:30,497 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 23:24:30,498 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 23:24:30,499 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 23:24:30,837 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 23:24:31,838 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 23:24:34,104 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:24:34,500 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:24:39,105 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:24:39,500 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:24:43,522 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 23:24:43,523 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 23:24:44,691 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:24:48,569 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 23:24:49,127 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:24:49,128 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:24:50,571 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:24:54,128 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:24:54,128 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:24:55,576 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:24:58,531 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 23:24:58,532 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 23:25:00,693 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:25:00,873 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 23:25:00,875 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 23:25:00,875 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 23:25:00,876 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 23:25:01,868 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 23:25:01,868 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 23:25:04,284 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:25:05,877 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:25:09,285 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:25:09,285 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:25:10,878 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:25:13,526 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 23:25:13,527 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 23:25:16,613 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:25:18,571 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 23:25:19,286 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:25:19,287 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:25:22,573 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:25:24,287 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:25:24,287 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:25:27,574 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:25:28,532 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 23:25:28,538 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 23:25:32,613 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:25:33,829 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 23:25:33,830 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 23:25:33,831 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 23:25:33,833 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 23:25:33,894 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 23:25:34,297 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:25:34,297 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:25:35,896 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 23:25:37,834 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:25:39,304 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:25:42,834 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:25:43,529 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 23:25:43,529 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 23:25:48,575 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 23:25:48,582 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:25:49,299 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:25:53,578 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:25:54,299 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:25:58,529 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 23:25:58,530 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 23:25:58,688 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:26:03,693 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:26:04,306 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:26:06,242 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 23:26:06,243 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 23:26:06,244 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 23:26:06,245 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 23:26:06,925 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 23:26:07,926 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 23:26:09,246 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:26:09,651 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:26:09,652 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:26:13,529 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 23:26:13,530 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 23:26:14,626 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:26:18,579 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 23:26:18,653 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:26:18,653 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:26:20,581 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:26:23,733 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:26:23,733 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:26:25,583 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:26:28,529 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 23:26:28,530 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 23:26:30,659 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:26:33,740 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:26:35,672 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:26:37,666 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 23:26:37,667 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 23:26:37,667 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 23:26:37,669 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 23:26:37,950 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 23:26:38,817 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:26:39,952 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 23:26:40,670 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:26:43,529 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 23:26:43,530 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 23:26:45,692 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:26:48,583 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 23:26:48,819 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:26:51,585 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:26:53,820 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:26:53,820 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:26:56,586 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:26:58,530 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 23:26:58,530 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 23:27:01,643 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:27:03,831 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:27:06,652 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:27:07,987 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 23:27:07,990 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 23:27:07,990 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 23:27:07,992 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 23:27:08,829 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:27:08,977 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 23:27:09,978 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 23:27:11,993 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:27:13,530 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 23:27:13,530 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 23:27:17,689 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:27:18,587 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 23:27:18,830 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:27:23,590 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:27:23,831 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:27:28,530 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 23:27:28,530 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 23:27:28,617 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:27:33,617 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:27:33,832 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:27:38,620 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:27:38,842 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:27:40,148 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 23:27:40,150 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 23:27:40,150 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 23:27:40,152 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 23:27:41,012 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 23:27:42,013 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 23:27:43,530 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 23:27:43,531 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 23:27:43,688 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:27:48,591 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 23:27:48,842 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:27:48,843 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:27:49,593 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:27:53,843 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:27:53,843 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:27:54,595 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:27:58,530 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 23:27:58,531 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 23:27:59,652 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:28:03,845 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:28:04,653 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:28:08,855 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:28:09,660 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:28:12,977 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 23:28:12,979 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 23:28:12,980 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 23:28:12,981 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 23:28:13,037 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 23:28:13,530 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 23:28:13,531 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 23:28:14,039 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 23:28:14,685 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:28:18,595 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 23:28:18,855 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:28:20,597 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:28:23,856 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:28:25,598 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:28:28,530 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 23:28:28,531 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 23:28:30,666 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:28:33,857 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:28:33,858 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:28:35,672 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:28:38,868 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:28:40,416 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 23:28:40,417 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 23:28:40,418 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 23:28:40,419 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 23:28:41,065 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 23:28:41,420 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:28:42,066 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 23:28:43,530 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 23:28:43,531 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 23:28:46,651 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:28:48,599 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 23:28:49,376 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:28:49,376 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:28:52,601 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:28:54,377 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:28:54,377 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:28:57,602 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:28:58,531 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 23:28:58,531 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 23:29:02,606 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:29:04,379 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:29:04,379 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:29:07,613 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:29:09,389 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:29:10,596 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 23:29:10,598 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 23:29:10,599 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 23:29:10,601 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 23:29:11,091 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 23:29:12,092 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 23:29:13,531 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 23:29:13,531 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 23:29:13,619 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:29:18,603 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 23:29:19,394 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:29:19,395 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:29:19,605 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:29:24,395 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:29:24,396 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:29:24,605 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:29:28,531 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 23:29:28,532 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 23:29:29,689 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:29:34,397 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:29:34,398 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:29:34,689 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:29:39,407 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:29:39,697 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:29:43,539 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 23:29:43,539 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 23:29:45,133 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 23:29:45,134 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 23:29:45,135 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 23:29:45,135 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:29:45,136 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 23:29:46,123 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 23:29:46,123 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 23:29:48,607 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 23:29:49,498 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:29:49,499 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:29:50,609 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:29:54,500 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:29:54,500 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:29:55,610 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:29:58,531 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 23:29:58,532 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 23:30:00,643 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:30:04,501 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:30:04,502 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:30:05,644 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:30:09,502 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:30:09,503 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:30:10,644 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:30:13,544 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 23:30:13,554 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 23:30:16,627 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:30:18,608 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 23:30:18,842 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 23:30:18,843 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 23:30:18,844 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 23:30:18,845 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 23:30:19,152 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 23:30:19,509 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:30:20,153 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 23:30:21,846 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:30:24,510 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:30:26,847 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:30:28,535 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 23:30:28,535 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 23:30:32,674 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:30:34,511 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:30:37,675 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:30:39,512 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:30:42,675 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:30:43,540 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 23:30:43,541 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 23:30:47,709 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:30:48,612 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 23:30:49,195 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 23:30:49,197 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 23:30:49,197 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 23:30:49,199 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 23:30:49,597 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:30:49,598 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:30:50,182 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 23:30:50,182 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 23:30:53,200 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:30:54,598 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:30:58,200 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:30:58,537 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 23:30:58,537 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 23:31:03,600 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:31:03,614 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:31:08,600 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:31:08,615 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:31:13,537 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 23:31:13,538 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 23:31:13,721 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:31:18,616 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:31:18,617 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 23:31:19,620 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:31:22,096 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 23:31:22,098 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 23:31:22,098 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 23:31:22,099 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 23:31:22,213 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 23:31:23,610 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:31:23,610 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:31:24,215 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 23:31:25,100 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:31:28,537 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 23:31:28,538 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 23:31:30,701 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:31:33,611 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:31:33,612 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:31:35,701 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:31:38,636 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:31:38,636 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:31:40,702 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:31:43,537 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 23:31:43,538 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 23:31:46,622 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:31:48,621 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 23:31:48,637 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:31:48,638 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:31:51,631 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:31:53,646 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:31:55,142 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 23:31:55,143 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 23:31:55,144 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 23:31:55,145 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 23:31:55,238 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 23:31:56,239 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 23:31:57,146 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:31:58,537 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 23:31:58,538 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 23:32:02,683 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:32:03,644 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:32:07,683 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:32:08,645 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:32:12,684 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:32:13,538 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 23:32:13,538 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 23:32:18,609 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:32:18,625 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 23:32:18,646 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:32:23,631 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:32:23,655 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:32:27,164 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 23:32:27,165 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 23:32:27,166 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 23:32:27,167 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 23:32:27,267 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 23:32:28,268 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 23:32:28,538 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 23:32:28,538 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 23:32:28,683 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:32:33,655 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:32:33,683 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:32:38,656 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:32:38,684 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:32:43,538 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 23:32:43,539 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 23:32:43,700 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:32:48,629 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 23:32:48,658 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:32:49,631 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:32:53,659 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:32:53,659 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:32:54,635 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:32:58,546 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 23:32:58,550 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 23:32:59,694 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:33:00,223 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 23:33:00,224 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 23:33:00,225 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 23:33:00,226 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 23:33:00,301 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 23:33:02,303 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 23:33:03,664 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:33:05,228 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:33:08,665 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:33:10,229 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:33:13,544 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 23:33:13,545 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 23:33:15,653 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:33:18,633 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 23:33:18,789 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:33:21,635 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:33:23,789 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:33:26,647 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:33:28,550 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 23:33:28,550 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 23:33:30,498 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 23:33:30,500 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 23:33:30,500 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 23:33:30,502 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 23:33:31,332 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 23:33:32,333 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 23:33:32,503 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:33:33,897 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:33:37,504 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:33:38,897 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:33:38,898 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:33:42,504 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:33:43,547 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 23:33:43,548 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 23:33:47,659 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:33:48,637 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 23:33:48,899 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:33:53,639 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:33:53,900 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:33:58,550 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 23:33:58,551 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 23:33:59,618 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:34:00,436 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 23:34:00,437 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 23:34:00,438 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 23:34:00,440 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 23:34:01,354 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 23:34:02,355 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 23:34:03,907 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:34:05,441 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:34:08,907 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:34:10,442 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:34:13,550 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 23:34:13,551 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 23:34:15,711 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:34:18,641 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 23:34:18,909 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:34:21,643 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:34:23,909 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:34:26,654 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:34:28,560 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 23:34:28,560 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 23:34:30,781 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 23:34:30,782 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 23:34:30,783 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 23:34:30,784 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 23:34:31,380 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 23:34:31,785 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:34:33,381 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 23:34:33,926 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:34:36,786 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:34:38,927 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:34:41,787 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:34:43,554 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 23:34:43,555 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 23:34:47,654 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:34:48,645 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 23:34:48,928 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:34:53,657 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:34:53,942 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:34:53,942 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:34:58,554 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 23:34:58,555 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 23:34:58,712 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:34:58,869 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 23:34:58,871 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 23:34:58,871 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 23:34:58,873 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 23:34:59,405 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 23:35:01,407 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 23:35:03,874 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:35:04,015 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:35:04,015 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:35:08,875 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:35:09,016 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:35:09,016 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:35:13,554 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 23:35:13,555 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 23:35:14,702 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:35:18,649 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 23:35:19,017 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:35:19,017 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:35:20,651 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:35:24,018 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:35:25,655 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:35:28,555 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 23:35:28,556 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 23:35:28,688 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 23:35:28,690 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 23:35:28,690 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 23:35:28,692 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 23:35:29,431 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 23:35:30,693 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:35:31,433 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 23:35:34,101 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:35:34,141 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:35:35,693 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:35:39,102 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:35:39,102 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:35:40,694 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:35:43,555 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 23:35:43,555 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 23:35:46,650 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:35:48,654 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 23:35:49,245 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:35:49,246 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:35:51,656 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:35:54,254 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:35:56,663 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:35:58,555 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 23:35:58,556 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 23:35:58,894 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 23:35:58,895 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 23:35:58,895 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 23:35:58,896 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 23:35:59,460 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 23:36:01,461 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 23:36:01,897 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:36:04,299 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:36:04,300 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:36:06,898 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:36:09,300 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:36:11,899 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:36:13,555 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 23:36:13,556 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 23:36:17,717 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:36:18,657 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 23:36:19,310 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:36:19,310 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:36:23,660 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:36:24,320 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:36:28,558 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 23:36:28,561 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 23:36:28,668 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:36:29,936 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 23:36:29,938 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 23:36:29,938 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 23:36:29,940 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 23:36:30,494 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 23:36:31,495 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 23:36:33,941 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:36:34,346 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:36:38,941 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:36:39,347 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:36:39,347 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:36:43,558 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 23:36:43,559 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 23:36:44,695 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:36:48,661 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 23:36:49,348 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:36:49,349 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:36:50,663 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:36:54,357 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:36:55,667 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:36:58,565 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 23:36:58,569 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 23:36:59,771 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 23:36:59,772 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 23:36:59,773 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 23:36:59,774 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 23:37:00,520 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 23:37:00,775 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:37:01,521 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 23:37:04,356 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:37:05,776 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:37:09,356 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:37:10,777 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:37:13,564 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 23:37:13,565 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 23:37:16,677 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:37:18,664 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 23:37:19,358 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:37:22,666 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:37:24,361 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:37:27,673 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:37:28,566 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 23:37:28,566 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 23:37:29,273 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 23:37:29,274 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 23:37:29,274 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 23:37:29,276 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 23:37:29,546 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 23:37:31,548 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 23:37:33,277 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:37:33,670 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:37:33,670 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:37:38,277 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:37:38,671 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:37:43,278 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:37:43,566 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 23:37:43,567 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 23:37:48,668 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 23:37:48,674 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:37:48,675 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:37:48,675 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:37:53,678 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:37:53,679 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:37:58,559 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 23:37:58,561 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 23:37:58,561 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 23:37:58,562 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 23:37:58,576 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 23:37:58,706 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:37:58,961 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:37:58,964 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 23:37:58,965 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 23:37:59,043 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:38:01,578 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 23:38:04,043 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:38:04,962 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:38:09,044 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:38:09,962 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:38:09,963 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:38:13,707 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 23:38:13,708 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 23:38:14,797 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:38:18,672 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 23:38:18,994 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:38:20,674 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:38:24,003 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:38:25,684 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:38:28,710 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 23:38:28,710 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 23:38:29,557 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 23:38:29,558 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 23:38:29,558 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 23:38:29,559 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 23:38:29,601 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 23:38:31,560 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:38:31,603 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 23:38:34,002 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:38:36,561 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:38:39,002 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:38:41,562 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:38:43,710 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 23:38:43,711 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 23:38:46,859 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:38:48,676 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 23:38:49,004 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:38:52,678 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:38:54,013 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:38:57,685 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:38:58,713 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 23:38:58,713 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 23:38:59,510 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 23:38:59,512 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 23:38:59,512 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 23:38:59,514 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 23:38:59,635 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 23:39:01,636 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 23:39:03,515 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:39:04,012 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:39:08,515 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:39:09,013 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:39:13,516 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:39:13,713 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 23:39:13,713 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 23:39:18,680 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 23:39:18,687 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:39:19,045 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:39:19,045 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:39:23,683 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:39:24,045 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:39:24,046 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:39:28,689 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:39:28,724 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 23:39:28,725 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 23:39:32,108 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 23:39:32,109 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 23:39:32,110 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 23:39:32,111 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 23:39:32,670 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 23:39:33,671 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 23:39:34,051 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:39:34,112 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:39:39,051 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:39:39,113 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:39:43,717 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 23:39:43,718 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 23:39:44,880 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:39:48,684 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 23:39:49,053 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:39:50,686 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:39:54,053 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:39:55,694 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:39:58,720 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 23:39:58,725 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 23:40:00,610 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 23:40:00,611 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 23:40:00,612 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 23:40:00,613 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 23:40:00,694 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 23:40:01,614 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:40:01,695 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 23:40:04,064 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:40:06,614 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:40:09,064 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:40:11,615 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:40:13,720 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 23:40:13,720 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 23:40:16,883 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:40:18,688 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 23:40:19,066 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:40:22,690 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:40:24,066 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:40:27,694 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:40:28,729 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 23:40:28,733 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 23:40:32,337 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 23:40:32,338 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 23:40:32,338 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 23:40:32,339 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 23:40:32,722 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 23:40:33,340 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:40:33,723 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 23:40:34,077 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:40:38,340 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:40:39,078 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:40:43,341 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:40:43,724 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 23:40:43,725 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 23:40:48,692 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 23:40:48,698 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:40:49,080 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:40:49,080 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:40:53,695 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:40:54,081 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:40:54,081 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:40:58,699 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:40:58,730 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 23:40:58,731 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 23:41:03,315 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 23:41:03,316 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 23:41:03,316 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 23:41:03,317 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 23:41:03,753 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 23:41:04,092 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:41:04,318 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:41:05,755 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 23:41:09,093 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:41:09,318 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:41:13,728 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 23:41:13,729 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 23:41:14,891 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:41:18,697 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 23:41:19,094 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:41:20,699 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:41:24,094 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:41:25,700 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:41:28,733 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 23:41:28,733 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 23:41:30,842 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:41:32,671 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 23:41:32,673 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 23:41:32,673 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 23:41:32,674 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 23:41:32,774 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 23:41:33,775 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 23:41:34,152 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:41:34,153 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:41:36,675 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:41:39,153 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:41:39,153 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:41:41,676 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:41:43,731 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 23:41:43,732 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 23:41:46,871 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:41:48,700 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 23:41:49,269 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:41:49,269 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:41:52,702 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:41:54,269 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:41:54,270 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:41:57,708 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:41:58,738 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 23:41:58,742 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 23:42:01,681 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 23:42:01,682 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 23:42:01,682 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 23:42:01,683 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 23:42:01,802 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 23:42:03,684 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:42:03,806 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 23:42:04,276 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:42:04,277 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:42:08,685 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:42:09,277 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:42:09,278 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:42:13,685 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:42:13,735 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 23:42:13,735 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 23:42:18,704 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 23:42:18,710 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:42:19,279 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:42:23,707 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:42:24,280 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:42:28,713 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:42:28,741 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 23:42:28,742 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 23:42:33,695 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 23:42:33,696 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 23:42:33,697 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 23:42:33,698 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 23:42:33,837 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 23:42:34,286 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:42:34,699 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:42:35,839 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 23:42:39,287 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:42:39,700 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:42:43,738 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 23:42:43,739 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 23:42:44,822 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:42:48,708 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 23:42:49,289 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:42:49,289 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:42:50,710 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:42:54,289 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:42:54,290 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:42:55,711 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:42:58,744 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 23:42:58,750 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 23:43:00,833 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:43:04,242 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 23:43:04,244 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 23:43:04,244 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 23:43:04,245 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 23:43:04,388 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:43:04,645 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:43:04,867 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 23:43:05,868 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 23:43:06,246 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:43:09,646 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:43:11,247 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:43:13,744 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 23:43:13,744 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 23:43:16,853 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:43:18,712 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 23:43:19,647 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:43:19,648 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:43:22,715 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:43:24,648 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:43:27,717 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:43:28,751 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 23:43:28,755 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 23:43:32,890 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:43:33,864 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 23:43:33,865 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 23:43:33,866 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 23:43:33,867 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 23:43:33,899 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 23:43:34,656 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:43:35,901 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 23:43:38,868 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:43:39,656 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:43:43,750 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 23:43:43,751 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 23:43:44,814 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:43:48,717 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 23:43:49,663 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:43:49,663 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:43:50,719 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:43:54,664 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:43:54,664 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:43:55,720 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:43:58,757 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 23:43:58,759 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 23:44:00,848 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:44:04,127 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 23:44:04,128 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 23:44:04,129 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 23:44:04,130 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 23:44:04,672 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:44:04,673 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:44:04,927 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 23:44:05,928 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 23:44:06,131 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:44:09,673 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:44:11,132 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:44:13,755 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 23:44:13,755 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 23:44:16,904 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:44:18,720 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 23:44:19,675 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:44:19,675 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:44:22,723 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:44:24,675 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:44:24,676 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:44:27,723 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:44:28,755 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 23:44:28,755 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 23:44:32,889 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:44:34,684 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:44:36,500 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 23:44:36,502 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 23:44:36,502 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 23:44:36,504 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 23:44:36,952 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 23:44:37,953 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 23:44:38,505 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:44:39,892 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:44:39,892 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:44:43,506 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:44:43,755 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 23:44:43,756 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 23:44:48,724 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 23:44:48,730 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:44:48,893 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:44:48,893 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:44:53,727 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:44:53,894 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:44:53,895 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:44:58,728 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:44:58,755 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 23:44:58,756 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 23:45:03,901 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:45:03,920 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:45:08,903 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:45:08,921 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:45:09,353 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 23:45:09,354 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 23:45:09,354 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 23:45:09,356 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 23:45:09,977 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 23:45:11,978 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 23:45:13,756 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 23:45:13,756 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 23:45:14,916 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:45:18,728 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 23:45:18,904 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:45:20,730 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:45:23,905 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:45:25,731 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:45:28,755 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 23:45:28,756 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 23:45:30,829 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:45:33,917 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:45:35,838 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:45:38,915 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:45:39,052 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 23:45:39,053 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 23:45:39,054 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 23:45:39,055 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 23:45:40,005 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 23:45:40,006 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 23:45:41,055 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:45:43,756 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 23:45:43,756 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 23:45:46,904 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:45:48,732 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 23:45:48,917 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:45:52,734 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:45:53,918 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:45:57,735 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:45:58,756 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 23:45:58,756 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 23:46:02,910 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:46:03,919 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:46:07,921 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:46:08,927 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:46:11,455 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 23:46:11,456 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 23:46:11,457 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 23:46:11,458 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 23:46:12,034 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 23:46:13,459 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:46:13,756 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 23:46:13,756 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 23:46:14,036 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 23:46:18,737 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 23:46:18,743 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:46:19,401 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:46:23,739 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:46:24,402 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:46:24,402 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:46:28,740 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:46:28,756 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 23:46:28,757 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 23:46:33,920 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:46:34,404 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:46:34,404 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:46:38,930 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:46:39,404 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:46:43,763 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 23:46:43,763 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 23:46:44,369 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 23:46:44,370 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 23:46:44,371 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 23:46:44,371 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:46:44,373 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 23:46:45,064 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 23:46:46,065 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 23:46:48,740 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 23:46:49,524 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:46:49,525 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:46:49,742 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:46:54,525 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:46:54,526 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:46:54,743 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:46:58,760 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 23:46:58,761 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 23:46:59,913 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:47:04,527 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:47:04,527 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:47:04,913 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:47:09,534 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:47:09,920 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:47:11,646 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 23:47:11,647 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 23:47:11,647 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 23:47:11,648 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 23:47:12,091 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 23:47:13,760 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 23:47:13,761 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 23:47:14,093 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 23:47:15,858 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:47:18,743 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 23:47:18,802 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:47:18,803 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:47:21,745 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:47:23,803 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:47:23,803 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:47:26,745 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:47:28,760 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 23:47:28,761 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 23:47:31,887 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:47:33,805 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:47:33,805 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:47:36,890 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:47:38,815 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:47:41,893 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:47:41,924 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 23:47:41,926 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 23:47:41,927 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 23:47:41,928 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 23:47:42,117 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 23:47:43,761 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 23:47:43,761 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 23:47:44,119 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 23:47:47,841 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:47:48,747 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 23:47:48,927 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:47:53,749 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:47:53,928 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:47:58,750 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:47:58,761 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 23:47:58,761 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 23:48:03,908 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:48:03,929 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:48:03,929 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:48:08,910 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:48:08,935 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:48:13,647 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 23:48:13,648 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 23:48:13,649 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 23:48:13,650 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 23:48:14,048 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:48:14,048 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:48:14,049 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 23:48:14,050 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 23:48:14,147 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 23:48:14,182 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:48:16,148 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 23:48:18,751 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 23:48:19,049 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:48:19,753 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:48:24,050 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:48:24,754 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:48:28,799 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 23:48:28,799 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 23:48:29,928 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:48:34,051 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:48:34,052 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:48:34,929 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:48:39,052 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:48:39,053 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:48:39,938 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:48:43,806 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 23:48:43,806 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 23:48:44,956 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:48:45,326 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 23:48:45,327 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 23:48:45,327 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 23:48:45,328 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 23:48:46,173 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 23:48:46,173 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 23:48:48,755 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 23:48:49,065 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:48:50,757 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:48:54,065 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:48:55,758 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:48:58,804 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 23:48:58,805 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 23:49:00,875 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:49:04,067 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:49:05,876 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:49:09,068 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:49:10,880 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:49:13,811 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 23:49:13,812 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 23:49:15,258 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 23:49:15,260 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 23:49:15,261 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 23:49:15,262 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 23:49:16,200 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 23:49:16,200 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 23:49:16,263 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:49:18,759 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 23:49:19,075 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:49:19,174 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:49:21,762 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:49:24,175 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:49:24,175 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:49:26,763 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:49:28,808 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 23:49:28,809 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 23:49:31,902 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:49:34,177 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:49:34,177 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:49:36,903 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:49:39,178 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:49:39,178 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:49:41,905 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:49:43,814 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 23:49:43,815 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 23:49:45,104 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 23:49:45,106 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 23:49:45,106 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 23:49:45,108 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 23:49:45,230 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 23:49:46,231 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 23:49:47,109 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:49:48,763 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 23:49:49,505 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:49:49,505 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:49:52,765 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:49:54,506 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:49:54,507 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:49:57,766 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:49:58,812 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 23:49:58,812 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 23:50:02,923 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:50:04,508 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:50:04,508 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:50:07,924 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:50:09,509 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:50:09,509 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:50:12,928 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:50:13,818 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 23:50:13,822 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 23:50:17,165 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 23:50:17,166 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 23:50:17,166 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 23:50:17,168 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 23:50:17,253 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 23:50:18,169 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:50:18,255 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 23:50:18,767 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 23:50:19,564 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:50:23,769 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:50:24,565 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:50:28,770 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:50:28,815 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 23:50:28,815 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 23:50:33,918 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:50:34,566 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:50:38,918 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:50:39,567 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:50:43,823 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 23:50:43,824 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 23:50:43,944 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:50:47,591 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 23:50:47,592 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 23:50:47,593 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 23:50:47,594 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 23:50:48,284 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 23:50:48,285 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 23:50:48,771 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 23:50:49,574 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:50:49,773 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:50:54,574 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:50:54,774 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:50:58,821 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 23:50:58,822 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 23:50:59,953 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:51:04,576 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:51:04,954 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:51:09,577 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:51:09,954 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:51:13,821 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 23:51:13,824 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 23:51:15,905 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:51:18,660 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 23:51:18,662 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 23:51:18,662 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 23:51:18,664 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 23:51:18,775 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 23:51:19,312 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 23:51:19,585 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:51:20,314 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 23:51:21,777 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:51:24,586 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:51:26,778 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:51:28,822 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 23:51:28,822 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 23:51:31,899 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:51:34,588 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:51:36,900 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:51:39,588 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:51:39,589 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:51:41,910 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:51:43,827 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 23:51:43,831 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 23:51:46,981 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:51:47,469 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 23:51:47,471 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 23:51:47,472 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 23:51:47,473 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 23:51:48,338 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 23:51:48,339 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 23:51:48,780 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 23:51:49,615 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:51:52,782 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:51:54,616 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:51:54,616 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:51:57,783 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:51:58,827 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 23:51:58,827 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 23:52:02,928 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:52:04,618 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:52:04,618 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:52:07,929 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:52:09,618 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:52:09,618 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:52:12,932 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:52:13,835 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 23:52:13,841 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 23:52:18,009 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:52:18,782 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 23:52:19,208 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 23:52:19,209 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 23:52:19,209 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 23:52:19,210 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 23:52:19,366 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 23:52:19,622 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:52:20,367 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 23:52:23,211 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:52:24,623 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:52:28,212 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:52:28,830 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 23:52:28,830 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 23:52:33,968 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:52:34,640 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:52:38,969 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:52:39,641 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:52:39,641 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:52:43,830 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 23:52:43,831 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 23:52:44,958 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:52:48,786 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 23:52:49,650 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:52:49,908 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 23:52:49,909 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 23:52:49,909 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 23:52:49,911 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 23:52:50,388 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 23:52:50,911 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:52:52,390 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 23:52:54,651 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:52:55,912 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:52:58,830 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 23:52:58,831 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 23:53:00,989 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:53:04,701 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:53:04,701 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:53:05,989 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:53:09,701 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:53:09,702 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:53:10,990 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:53:13,836 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 23:53:13,837 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 23:53:16,965 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:53:18,790 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 23:53:18,859 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 23:53:18,861 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 23:53:18,862 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 23:53:18,863 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 23:53:19,414 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 23:53:19,710 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:53:20,415 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 23:53:22,865 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:53:24,711 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:53:27,865 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:53:28,836 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 23:53:28,836 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 23:53:32,951 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:53:34,712 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:53:37,951 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:53:39,713 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:53:42,952 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:53:43,836 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 23:53:43,837 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 23:53:47,985 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:53:48,798 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 23:53:49,721 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:53:53,336 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 23:53:53,337 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 23:53:53,338 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 23:53:53,338 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:53:53,339 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 23:53:53,447 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 23:53:54,448 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 23:53:54,736 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:53:58,340 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:53:58,836 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 23:53:58,837 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 23:54:04,003 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:54:04,737 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:54:09,004 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:54:09,738 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:54:13,836 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 23:54:13,837 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 23:54:14,937 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:54:18,799 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 23:54:19,739 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:54:20,803 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:54:24,744 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:54:25,667 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 23:54:25,669 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 23:54:25,669 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 23:54:25,670 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 23:54:26,476 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 23:54:26,476 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 23:54:26,671 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:54:28,837 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 23:54:28,837 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 23:54:31,929 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:54:34,814 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:54:36,929 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:54:39,907 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:54:39,907 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:54:41,930 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:54:43,836 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 23:54:43,837 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 23:54:46,999 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:54:48,804 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 23:54:48,908 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:54:48,909 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:54:52,812 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:54:53,922 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:54:55,262 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 23:54:55,264 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 23:54:55,264 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 23:54:55,266 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 23:54:55,501 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 23:54:56,503 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 23:54:58,267 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:54:58,837 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 23:54:58,837 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 23:55:03,917 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:55:03,996 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:55:08,945 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:55:08,945 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:55:08,996 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:55:13,837 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 23:55:13,837 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 23:55:14,935 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:55:18,807 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 23:55:18,947 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:55:20,809 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:55:23,954 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:55:25,819 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:55:27,207 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 23:55:27,209 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 23:55:27,209 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 23:55:27,211 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 23:55:27,533 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 23:55:28,534 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 23:55:28,837 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 23:55:28,838 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 23:55:30,918 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:55:33,956 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:55:35,919 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:55:38,957 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:55:40,919 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:55:43,837 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 23:55:43,838 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 23:55:46,012 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:55:48,811 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 23:55:49,371 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:55:49,371 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:55:51,814 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:55:54,382 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:55:56,819 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:55:58,670 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 23:55:58,670 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 23:55:58,671 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 23:55:58,672 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 23:55:59,071 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 23:55:59,072 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 23:55:59,558 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 23:56:00,559 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 23:56:02,227 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:56:04,382 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:56:07,227 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:56:09,382 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:56:12,228 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:56:14,071 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 23:56:14,071 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 23:56:18,178 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:56:18,815 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 23:56:19,384 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:56:23,822 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:56:24,393 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:56:27,761 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 23:56:27,761 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 23:56:27,762 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 23:56:27,763 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 23:56:28,585 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 23:56:29,071 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 23:56:29,071 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 23:56:29,153 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:56:29,587 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 23:56:34,153 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:56:34,392 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:56:39,154 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:56:39,393 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:56:44,071 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 23:56:44,072 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 23:56:45,144 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:56:48,819 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 23:56:49,394 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:56:50,821 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:56:54,398 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:56:55,830 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:56:57,402 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 23:56:57,404 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 23:56:57,404 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 23:56:57,406 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 23:56:57,608 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 23:56:59,071 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 23:56:59,072 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 23:56:59,609 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 23:57:01,212 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:57:04,399 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:57:06,213 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:57:09,400 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:57:11,214 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:57:14,071 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 23:57:14,072 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 23:57:17,138 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:57:18,824 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 23:57:19,402 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:57:22,827 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:57:24,404 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:57:27,837 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:57:29,072 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 23:57:29,072 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 23:57:29,116 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 23:57:29,166 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 23:57:29,166 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 23:57:29,168 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 23:57:29,638 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 23:57:31,639 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 23:57:33,170 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:57:34,517 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:57:38,170 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:57:39,518 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:57:43,171 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:57:44,071 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 23:57:44,072 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 23:57:48,199 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:57:48,827 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 23:57:49,519 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:57:53,830 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:57:54,521 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:57:58,831 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:57:59,071 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 23:57:59,072 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 23:58:04,145 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:58:04,526 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:58:04,841 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 23:58:04,843 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 23:58:04,843 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 23:58:04,844 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 23:58:05,669 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 23:58:07,671 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 23:58:09,527 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:58:09,845 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:58:14,072 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 23:58:14,072 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 23:58:15,217 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:58:18,831 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 23:58:19,528 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:58:20,834 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:58:24,530 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:58:25,834 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:58:29,083 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 23:58:29,084 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 23:58:31,244 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:58:34,241 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 23:58:34,242 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 23:58:34,243 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 23:58:34,244 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 23:58:34,647 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:58:34,689 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 23:58:35,690 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 23:58:36,245 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:58:39,648 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:58:41,245 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:58:44,075 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 23:58:44,076 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 23:58:47,235 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:58:48,835 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 23:58:49,649 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:58:52,838 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:58:54,650 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:58:57,838 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:58:59,075 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 23:58:59,076 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 23:59:03,197 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:59:04,657 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:59:05,361 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 23:59:05,362 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 23:59:05,363 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 23:59:05,364 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 23:59:05,720 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 23:59:07,722 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 23:59:08,366 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:59:09,767 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:59:13,366 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:59:14,075 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 23:59:14,076 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 23:59:18,838 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 23:59:18,844 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:59:19,768 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:59:23,841 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:59:24,770 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:59:28,841 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:59:29,075 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 23:59:29,076 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 23:59:34,229 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:59:34,775 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:59:37,070 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-25 23:59:37,072 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-25 23:59:37,072 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-25 23:59:37,074 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-25 23:59:37,749 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-25 23:59:39,751 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-25 23:59:39,776 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:59:40,075 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:59:44,076 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 23:59:44,076 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-25 23:59:45,221 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:59:48,842 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-25 23:59:49,808 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:59:50,845 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:59:54,809 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-25 23:59:55,845 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-25 23:59:59,076 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-25 23:59:59,076 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 00:00:01,155 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:00:04,821 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:00:06,161 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:00:06,287 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 00:00:06,289 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 00:00:06,289 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 00:00:06,291 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 00:00:06,774 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 00:00:07,775 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 00:00:09,815 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:00:11,292 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:00:14,076 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 00:00:14,076 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 00:00:17,198 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:00:18,847 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 00:00:19,817 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:00:22,849 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:00:24,817 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:00:27,850 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:00:29,076 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 00:00:29,076 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 00:00:33,171 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:00:34,826 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:00:38,029 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 00:00:38,031 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 00:00:38,032 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 00:00:38,033 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 00:00:38,800 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 00:00:39,034 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:00:39,801 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 00:00:39,826 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:00:44,035 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:00:44,076 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 00:00:44,077 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 00:00:48,850 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 00:00:49,827 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:00:49,852 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:00:54,828 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:00:54,853 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:00:59,076 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 00:00:59,077 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 00:01:00,195 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:01:04,833 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:01:05,198 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:01:09,462 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 00:01:09,463 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 00:01:09,464 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 00:01:09,464 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 00:01:09,836 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 00:01:09,857 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:01:09,857 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:01:10,465 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:01:11,837 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 00:01:14,076 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 00:01:14,077 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 00:01:16,153 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:01:18,854 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 00:01:19,859 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:01:19,859 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:01:21,856 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:01:24,860 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:01:24,860 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:01:26,857 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:01:29,076 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 00:01:29,077 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 00:01:32,241 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:01:34,866 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:01:37,247 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:01:38,798 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 00:01:38,799 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 00:01:38,800 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 00:01:38,801 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 00:01:38,865 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 00:01:39,866 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 00:01:39,943 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:01:39,944 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:01:42,802 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:01:44,076 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 00:01:44,077 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 00:01:48,239 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:01:48,858 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 00:01:49,945 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:01:49,945 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:01:53,860 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:01:54,946 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:01:54,946 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:01:58,861 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:01:59,077 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 00:01:59,077 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 00:02:04,203 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:02:04,947 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:02:04,948 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:02:09,207 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:02:09,955 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:02:11,681 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 00:02:11,683 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 00:02:11,683 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 00:02:11,685 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 00:02:11,890 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 00:02:13,891 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 00:02:14,077 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 00:02:14,078 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 00:02:15,157 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:02:18,862 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 00:02:19,632 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:02:19,633 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:02:20,864 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:02:24,633 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:02:25,865 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:02:29,077 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 00:02:29,078 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 00:02:31,208 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:02:34,635 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:02:34,635 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:02:36,210 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:02:39,645 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:02:41,216 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:02:43,684 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 00:02:43,686 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 00:02:43,687 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 00:02:43,688 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 00:02:43,917 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 00:02:44,102 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 00:02:44,103 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 00:02:45,919 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 00:02:46,226 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:02:48,866 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 00:02:49,644 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:02:49,644 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:02:51,869 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:02:54,645 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:02:56,869 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:02:59,101 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 00:02:59,102 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 00:03:02,196 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:03:04,647 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:03:04,647 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:03:07,196 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:03:09,648 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:03:09,648 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:03:12,200 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:03:14,111 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 00:03:14,115 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 00:03:16,289 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 00:03:16,291 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 00:03:16,292 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 00:03:16,293 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 00:03:16,946 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 00:03:17,294 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:03:17,947 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 00:03:18,870 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 00:03:19,705 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:03:19,706 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:03:22,872 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:03:24,706 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:03:24,706 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:03:27,873 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:03:29,108 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 00:03:29,109 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 00:03:33,206 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:03:34,708 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:03:34,708 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:03:38,206 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:03:39,709 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:03:39,709 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:03:43,209 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:03:44,113 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 00:03:44,118 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 00:03:48,135 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 00:03:48,137 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 00:03:48,138 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 00:03:48,140 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 00:03:48,874 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 00:03:48,881 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:03:48,971 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 00:03:49,716 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:03:49,717 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:03:49,972 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 00:03:53,877 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:03:54,717 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:03:54,718 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:03:58,877 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:03:59,110 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 00:03:59,110 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 00:04:04,210 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:04:04,818 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:04:04,818 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:04:09,211 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:04:09,819 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:04:09,819 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:04:14,117 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 00:04:14,117 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 00:04:14,242 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:04:17,395 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 00:04:17,397 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 00:04:17,398 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 00:04:17,399 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 00:04:17,999 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 00:04:18,878 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 00:04:19,826 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:04:19,880 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:04:20,002 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 00:04:24,827 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:04:24,881 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:04:29,117 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 00:04:29,118 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 00:04:30,193 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:04:34,828 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:04:35,194 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:04:39,829 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:04:40,194 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:04:44,117 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 00:04:44,118 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 00:04:45,284 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:04:48,881 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 00:04:49,837 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:04:50,064 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 00:04:50,066 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 00:04:50,067 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 00:04:50,068 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 00:04:51,024 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 00:04:51,070 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:04:52,025 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 00:04:54,837 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:04:56,070 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:04:59,117 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 00:04:59,118 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 00:05:01,265 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:05:04,839 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:05:06,266 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:05:09,839 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:05:11,267 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:05:14,117 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 00:05:14,118 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 00:05:16,286 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:05:18,890 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 00:05:19,846 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:05:20,744 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 00:05:20,746 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 00:05:20,747 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 00:05:20,748 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 00:05:21,049 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 00:05:21,749 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:05:22,050 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 00:05:25,145 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:05:25,146 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:05:26,750 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:05:29,117 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 00:05:29,118 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 00:05:32,196 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:05:34,243 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:05:34,243 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:05:37,196 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:05:39,244 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:05:39,244 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:05:42,197 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:05:44,129 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 00:05:44,139 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 00:05:47,270 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:05:48,891 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 00:05:49,253 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:05:49,741 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 00:05:49,742 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 00:05:49,743 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 00:05:49,746 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 00:05:50,077 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 00:05:52,079 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 00:05:52,747 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:05:54,254 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:05:57,747 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:05:59,121 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 00:05:59,122 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 00:06:03,245 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:06:04,255 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:06:08,246 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:06:09,256 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:06:13,247 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:06:14,129 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 00:06:14,136 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 00:06:18,305 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:06:18,895 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 00:06:19,262 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:06:19,625 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 00:06:19,626 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 00:06:19,627 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 00:06:19,628 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 00:06:20,100 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 00:06:22,102 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 00:06:23,630 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:06:24,263 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:06:28,630 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:06:29,123 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 00:06:29,123 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 00:06:34,223 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:06:34,264 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:06:39,224 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:06:39,265 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:06:44,127 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 00:06:44,128 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 00:06:44,263 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:06:46,039 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 00:06:46,040 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 00:06:46,040 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 00:06:46,041 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 00:06:46,125 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 00:06:48,127 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 00:06:48,899 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 00:06:49,440 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:06:49,901 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:06:54,441 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:06:54,441 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:06:54,902 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:06:59,124 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 00:06:59,125 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 00:07:00,229 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:07:04,511 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:07:05,230 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:07:09,512 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:07:09,512 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:07:10,231 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:07:14,134 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 00:07:14,137 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 00:07:16,233 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:07:18,900 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 00:07:19,457 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 00:07:19,458 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 00:07:19,459 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 00:07:19,460 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 00:07:19,605 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:07:20,156 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 00:07:20,157 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 00:07:21,461 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:07:24,606 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:07:26,461 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:07:29,125 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 00:07:29,126 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 00:07:32,277 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:07:34,608 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:07:34,608 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:07:37,277 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:07:39,609 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:07:39,609 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:07:42,278 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:07:44,125 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 00:07:44,126 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 00:07:48,264 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:07:48,914 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 00:07:49,626 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:07:51,954 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 00:07:51,956 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 00:07:51,956 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 00:07:51,958 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 00:07:52,185 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 00:07:53,959 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:07:54,187 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 00:07:54,618 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:07:58,959 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:07:59,125 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 00:07:59,126 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 00:08:04,288 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:08:04,667 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:08:04,668 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:08:09,290 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:08:09,668 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:08:14,126 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 00:08:14,126 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 00:08:15,189 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:08:18,917 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 00:08:19,677 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:08:20,923 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:08:22,035 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 00:08:22,036 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 00:08:22,037 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 00:08:22,038 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 00:08:22,211 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 00:08:24,213 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 00:08:24,677 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:08:26,040 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:08:29,126 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 00:08:29,126 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 00:08:31,287 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:08:34,678 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:08:36,288 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:08:39,679 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:08:41,288 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:08:44,126 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 00:08:44,126 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 00:08:47,283 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:08:48,923 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 00:08:49,687 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:08:52,012 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 00:08:52,013 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 00:08:52,013 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 00:08:52,015 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 00:08:52,236 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 00:08:53,016 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:08:54,238 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 00:08:54,687 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:08:54,687 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:08:58,017 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:08:59,126 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 00:08:59,126 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 00:09:03,222 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:09:04,689 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:09:04,689 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:09:08,223 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:09:09,690 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:09:09,690 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:09:13,224 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:09:14,126 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 00:09:14,126 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 00:09:18,260 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:09:18,921 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 00:09:19,697 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:09:22,047 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 00:09:22,048 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 00:09:22,049 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 00:09:22,053 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 00:09:22,264 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 00:09:24,054 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:09:24,265 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 00:09:24,697 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:09:29,054 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:09:29,126 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 00:09:29,127 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 00:09:34,251 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:09:34,698 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:09:39,251 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:09:39,699 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:09:44,126 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 00:09:44,127 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 00:09:44,285 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:09:48,926 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 00:09:49,714 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:09:49,927 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:09:52,042 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 00:09:52,044 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 00:09:52,045 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 00:09:52,046 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 00:09:52,321 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 00:09:54,323 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 00:09:54,709 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:09:55,047 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:09:59,126 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 00:09:59,127 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 00:10:00,188 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:10:04,710 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:10:05,188 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:10:09,869 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:10:09,869 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:10:10,189 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:10:14,126 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 00:10:14,127 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 00:10:15,269 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:10:18,928 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 00:10:19,880 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:10:20,939 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:10:24,794 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 00:10:24,796 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 00:10:24,796 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 00:10:24,801 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 00:10:24,941 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:10:25,349 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 00:10:26,350 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 00:10:26,802 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:10:29,127 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 00:10:29,127 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 00:10:32,270 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:10:34,942 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:10:34,943 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:10:37,271 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:10:39,943 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:10:39,944 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:10:42,272 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:10:44,127 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 00:10:44,127 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 00:10:48,260 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:10:48,933 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 00:10:49,952 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:10:53,939 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:10:54,069 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 00:10:54,070 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 00:10:54,071 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 00:10:54,072 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 00:10:54,381 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 00:10:54,951 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:10:56,382 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 00:10:59,074 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:10:59,127 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 00:10:59,127 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 00:11:04,229 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:11:04,953 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:11:09,230 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:11:09,954 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:11:14,127 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 00:11:14,127 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 00:11:14,408 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:11:18,936 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 00:11:19,939 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:11:19,959 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:11:23,248 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 00:11:23,250 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 00:11:23,251 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 00:11:23,252 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 00:11:23,406 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 00:11:24,408 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 00:11:24,966 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:11:24,967 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:11:25,253 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:11:29,127 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 00:11:29,128 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 00:11:30,263 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:11:34,968 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:11:34,968 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:11:35,264 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:11:39,969 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:11:39,969 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:11:40,265 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:11:44,127 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 00:11:44,128 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 00:11:45,297 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:11:48,939 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 00:11:49,973 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:11:50,940 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:11:53,469 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 00:11:53,470 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 00:11:53,470 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 00:11:53,472 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 00:11:54,431 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 00:11:54,431 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 00:11:54,977 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:11:56,473 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:11:59,127 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 00:11:59,128 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 00:12:02,207 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:12:04,978 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:12:07,207 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:12:09,980 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:12:09,980 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:12:12,208 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:12:14,127 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 00:12:14,128 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 00:12:17,245 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:12:18,950 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 00:12:19,984 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:12:22,956 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:12:24,128 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 00:12:24,130 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 00:12:24,131 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 00:12:24,132 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 00:12:24,457 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 00:12:24,987 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:12:26,459 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 00:12:28,133 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:12:29,127 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 00:12:29,128 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 00:12:33,297 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:12:34,992 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:12:38,297 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:12:39,993 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:12:43,298 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:12:44,128 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 00:12:44,128 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 00:12:48,948 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 00:12:48,950 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:12:49,994 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:12:53,960 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:12:55,004 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:12:56,044 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 00:12:56,046 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 00:12:56,046 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 00:12:56,051 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 00:12:56,492 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 00:12:58,494 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 00:12:59,052 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:12:59,128 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 00:12:59,128 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 00:13:04,252 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:13:05,002 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:13:09,253 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:13:10,003 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:13:14,128 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 00:13:14,128 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 00:13:15,239 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:13:18,952 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 00:13:20,005 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:13:20,957 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:13:25,021 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:13:25,961 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:13:26,081 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 00:13:26,082 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 00:13:26,083 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 00:13:26,086 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 00:13:26,520 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 00:13:28,521 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 00:13:29,128 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 00:13:29,129 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 00:13:31,243 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:13:35,015 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:13:35,015 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:13:36,244 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:13:40,016 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:13:40,016 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:13:41,245 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:13:44,128 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 00:13:44,129 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 00:13:47,242 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:13:48,957 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 00:13:50,025 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:13:52,969 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:13:54,960 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 00:13:54,961 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 00:13:54,962 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 00:13:54,963 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 00:13:55,107 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:13:55,549 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 00:13:56,550 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 00:13:58,965 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:13:59,128 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 00:13:59,129 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 00:14:04,209 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:14:04,367 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:14:09,210 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:14:09,367 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:14:14,128 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 00:14:14,129 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 00:14:14,237 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:14:18,960 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 00:14:19,369 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:14:19,963 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:14:24,383 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:14:24,970 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:14:26,084 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 00:14:26,086 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 00:14:26,087 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 00:14:26,088 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 00:14:26,575 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 00:14:28,576 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 00:14:29,128 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 00:14:29,129 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 00:14:30,213 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:14:34,235 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:14:35,213 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:14:39,236 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:14:40,214 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:14:44,129 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 00:14:44,129 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 00:14:45,274 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:14:48,964 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 00:14:49,237 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:14:50,967 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:14:54,246 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:14:55,961 ERROR gpu :316822 [interfaces.py:monitor():142] Failed to sample metric: Not Supported +2024-12-26 00:14:55,969 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:14:56,498 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 00:14:56,500 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 00:14:56,500 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 00:14:56,501 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 00:14:56,601 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 00:14:58,603 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 00:14:59,129 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 00:14:59,129 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 00:15:01,244 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:15:04,245 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:15:06,244 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:15:09,246 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:15:11,245 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:15:14,129 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 00:15:14,129 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 00:15:17,204 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:15:18,972 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 00:15:19,260 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:15:20,739 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 00:15:20,740 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 00:15:20,740 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 00:15:20,742 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 00:15:21,628 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 00:15:22,629 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 00:15:22,742 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:15:24,258 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:15:27,743 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:15:29,129 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 00:15:29,130 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 00:15:33,278 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:15:34,260 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:15:38,278 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:15:39,308 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:15:43,279 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:15:44,129 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 00:15:44,130 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 00:15:48,290 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:15:48,973 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 00:15:49,318 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:15:53,985 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:15:54,319 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:15:54,764 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 00:15:54,766 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 00:15:54,766 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 00:15:54,767 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 00:15:55,658 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 00:15:56,659 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 00:15:59,129 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 00:15:59,130 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 00:15:59,204 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:16:04,204 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:16:04,321 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:16:09,205 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:16:09,321 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:16:14,129 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 00:16:14,130 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 00:16:14,237 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:16:18,981 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 00:16:19,329 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:16:19,985 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:16:23,571 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 00:16:23,572 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 00:16:23,573 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 00:16:23,575 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 00:16:23,687 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 00:16:24,334 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:16:24,688 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 00:16:25,576 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:16:29,129 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 00:16:29,130 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 00:16:31,280 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:16:34,336 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:16:36,281 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:16:39,336 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:16:41,281 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:16:44,129 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 00:16:44,130 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 00:16:47,266 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:16:48,981 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 00:16:49,345 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:16:51,933 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 00:16:51,936 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 00:16:51,936 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 00:16:51,939 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 00:16:52,713 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 00:16:52,713 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 00:16:52,940 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:16:54,344 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:16:57,941 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:16:59,129 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 00:16:59,130 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 00:17:03,294 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:17:04,346 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:17:08,295 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:17:09,346 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:17:13,296 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:17:14,130 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 00:17:14,130 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 00:17:18,992 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 00:17:18,993 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:17:19,355 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:17:20,253 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 00:17:20,255 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 00:17:20,255 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 00:17:20,257 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 00:17:20,747 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 00:17:20,748 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 00:17:24,258 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:17:24,628 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:17:29,130 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 00:17:29,131 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 00:17:30,192 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:17:34,630 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:17:35,192 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:17:39,631 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:17:40,193 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:17:44,130 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 00:17:44,131 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 00:17:45,299 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:17:48,998 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 00:17:49,644 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:17:51,002 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:17:53,128 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 00:17:53,130 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 00:17:53,131 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 00:17:53,132 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 00:17:53,778 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 00:17:54,640 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:17:54,779 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 00:17:56,133 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:17:59,130 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 00:17:59,131 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 00:18:01,274 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:18:04,641 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:18:06,275 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:18:09,642 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:18:11,276 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:18:14,130 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 00:18:14,131 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 00:18:16,296 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:18:18,997 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 00:18:19,649 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:18:22,006 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:18:24,129 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 00:18:24,131 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 00:18:24,131 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 00:18:24,137 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 00:18:24,648 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:18:24,809 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 00:18:25,810 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 00:18:27,138 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:18:29,130 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 00:18:29,131 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 00:18:32,219 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:18:34,651 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:18:37,220 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:18:39,652 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:18:42,220 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:18:44,130 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 00:18:44,131 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 00:18:48,208 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:18:49,011 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 00:18:49,659 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:18:54,018 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:18:54,144 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 00:18:54,146 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 00:18:54,146 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 00:18:54,149 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 00:18:54,658 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:18:54,834 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 00:18:55,835 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 00:18:59,130 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 00:18:59,131 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 00:18:59,301 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:19:04,302 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:19:04,659 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:19:09,302 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:19:09,660 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:19:14,131 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 00:19:14,131 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 00:19:15,243 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:19:19,013 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 00:19:19,674 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:19:21,023 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:19:24,205 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 00:19:24,207 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 00:19:24,207 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 00:19:24,211 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 00:19:24,666 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:19:24,870 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 00:19:25,871 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 00:19:26,212 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:19:29,131 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 00:19:29,131 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 00:19:32,202 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:19:34,667 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:19:37,203 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:19:39,668 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:19:42,203 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:19:44,131 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 00:19:44,131 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 00:19:48,103 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:19:49,012 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 00:19:49,238 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:19:49,325 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:19:49,885 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 00:19:54,015 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:19:56,701 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:19:56,961 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:19:59,015 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:20:04,016 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:20:05,234 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:20:05,653 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:20:05,695 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 00:20:05,696 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 00:20:09,787 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:20:13,601 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:20:13,898 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:20:14,788 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:20:14,982 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 00:20:14,982 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 00:20:19,021 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 00:20:20,023 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:20:22,072 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:20:22,341 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:20:25,023 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:20:30,024 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:20:30,272 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:20:30,447 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:20:30,926 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 00:20:30,927 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 00:20:35,067 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:20:39,255 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:20:40,068 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:20:43,417 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:20:45,273 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 00:20:45,274 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 00:20:45,364 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:20:45,433 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 00:20:47,435 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 00:20:49,020 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 00:20:50,418 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:20:51,023 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:20:55,419 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:20:56,024 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:21:00,273 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 00:21:00,274 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 00:21:01,437 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:21:05,421 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:21:05,422 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:21:06,438 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:21:10,422 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:21:10,423 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:21:11,439 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:21:15,274 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 00:21:15,274 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 00:21:17,373 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:21:19,023 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 00:21:20,424 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:21:23,025 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:21:25,425 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:21:28,026 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:21:30,274 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 00:21:30,274 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 00:21:33,435 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:21:35,426 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:21:38,436 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:21:43,436 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:21:48,437 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:21:49,057 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 00:21:54,059 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:21:59,061 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:21:59,343 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:21:59,343 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:21:59,344 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 00:21:59,344 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 00:22:04,345 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:22:04,345 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:22:04,503 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:22:09,346 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:22:09,347 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:22:09,503 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:22:14,340 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 00:22:14,340 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 00:22:15,501 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:22:19,031 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 00:22:19,348 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:22:19,348 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:22:21,033 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:22:24,349 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:22:24,349 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:22:26,034 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:22:29,341 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 00:22:29,341 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 00:22:31,407 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:22:34,351 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:22:34,351 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:22:36,408 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:22:39,352 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:22:39,352 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:22:41,408 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:22:44,340 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 00:22:44,341 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 00:22:46,453 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:22:49,039 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 00:22:49,353 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:22:49,354 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:22:52,042 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:22:54,355 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:22:54,355 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:22:57,043 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:22:59,340 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 00:22:59,341 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 00:23:02,469 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:23:04,356 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:23:04,357 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:23:07,470 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:23:09,357 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:23:09,358 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:23:12,470 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:23:14,340 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 00:23:14,341 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 00:23:18,460 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:23:19,039 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 00:23:19,359 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:23:19,359 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:23:24,042 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:23:24,361 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:23:24,361 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:23:29,042 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:23:29,341 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 00:23:29,341 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 00:23:34,363 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:23:34,363 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:23:34,456 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:23:39,364 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:23:39,364 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:23:39,457 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:23:44,341 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 00:23:44,341 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 00:23:45,447 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:23:49,043 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 00:23:49,366 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:23:49,366 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:23:51,045 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:23:54,367 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:23:54,367 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:23:56,046 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:24:01,047 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:24:06,051 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:24:11,053 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:24:16,053 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:24:19,047 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 00:24:22,050 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:24:27,050 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:24:32,051 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:24:35,989 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:24:35,994 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 00:24:35,995 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 00:24:37,125 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:24:39,760 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 00:24:40,995 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:24:40,995 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:24:42,394 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:24:43,762 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 00:24:45,996 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:24:45,996 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:24:47,394 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:24:49,052 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 00:24:50,989 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 00:24:50,989 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 00:24:53,133 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:24:55,998 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:24:55,998 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:24:58,134 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:25:00,999 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:25:01,000 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:25:03,135 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:25:05,999 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 00:25:06,003 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 00:25:09,096 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:25:09,242 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 00:25:09,244 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 00:25:09,244 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 00:25:09,246 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 00:25:09,785 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 00:25:11,006 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:25:11,786 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 00:25:14,247 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:25:16,007 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:25:19,055 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 00:25:20,057 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:25:20,994 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 00:25:20,995 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 00:25:25,121 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:25:26,009 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:25:26,009 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:25:30,122 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:25:31,010 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:25:31,010 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:25:35,129 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:25:36,001 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 00:25:36,007 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 00:25:39,665 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 00:25:39,666 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 00:25:39,667 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 00:25:39,668 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 00:25:39,813 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 00:25:40,669 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:25:41,069 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:25:41,069 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:25:41,815 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 00:25:45,670 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:25:46,070 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:25:46,070 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:25:49,059 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 00:25:50,998 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 00:25:50,999 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 00:25:51,080 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:25:56,072 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:25:56,072 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:25:56,081 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:26:01,073 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:26:01,073 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:26:01,081 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:26:06,008 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 00:26:06,013 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 00:26:06,129 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:26:08,060 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 00:26:08,062 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 00:26:08,062 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 00:26:08,063 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 00:26:08,837 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 00:26:09,839 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 00:26:11,213 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:26:12,065 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:26:16,213 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:26:17,065 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:26:19,063 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 00:26:21,160 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 00:26:21,160 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 00:26:22,232 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:26:26,215 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:26:27,232 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:26:31,216 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:26:32,233 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:26:36,079 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 00:26:36,080 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 00:26:38,214 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:26:38,625 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 00:26:38,626 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 00:26:38,627 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 00:26:38,628 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 00:26:38,868 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 00:26:39,869 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 00:26:41,233 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:26:43,630 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:26:46,234 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:26:48,630 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:26:49,067 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 00:26:51,073 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 00:26:51,074 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 00:26:54,158 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:26:56,236 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:26:59,158 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:27:01,236 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:27:04,159 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:27:06,081 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 00:27:06,082 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 00:27:09,234 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:27:10,846 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 00:27:10,848 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 00:27:10,848 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 00:27:10,850 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 00:27:10,898 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 00:27:11,900 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 00:27:11,973 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:27:11,973 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:27:14,851 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:27:16,976 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:27:16,976 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:27:19,072 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 00:27:20,074 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:27:21,077 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 00:27:21,077 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 00:27:25,171 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:27:26,977 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:27:30,172 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:27:31,978 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:27:35,172 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:27:36,081 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 00:27:36,081 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 00:27:40,190 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:27:40,908 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 00:27:40,909 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 00:27:40,910 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 00:27:40,911 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 00:27:40,926 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 00:27:41,928 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 00:27:42,065 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:27:45,912 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:27:47,066 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:27:47,066 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:27:49,075 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 00:27:51,077 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:27:51,081 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 00:27:51,081 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 00:27:56,204 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:27:57,067 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:27:57,068 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:28:01,205 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:28:02,069 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:28:02,069 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:28:06,081 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 00:28:06,082 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 00:28:07,205 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:28:12,078 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:28:12,213 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:28:12,972 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 00:28:12,973 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 00:28:12,974 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 00:28:12,975 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 00:28:13,953 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 00:28:13,954 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 00:28:17,415 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:28:17,415 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:28:17,976 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:28:19,079 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 00:28:21,081 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 00:28:21,082 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 00:28:23,190 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:28:26,416 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:28:26,417 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:28:28,190 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:28:31,417 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:28:31,417 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:28:33,191 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:28:36,081 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 00:28:36,082 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 00:28:38,252 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:28:41,429 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:28:43,160 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 00:28:43,161 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 00:28:43,162 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 00:28:43,163 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 00:28:43,979 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 00:28:43,979 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 00:28:44,164 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:28:46,566 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:28:49,083 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 00:28:50,085 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:28:51,081 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 00:28:51,082 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 00:28:55,229 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:28:56,568 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:29:00,230 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:29:01,569 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:29:05,230 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:29:06,082 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 00:29:06,082 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 00:29:11,220 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:29:11,580 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:29:11,581 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:29:14,167 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 00:29:14,168 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 00:29:14,168 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 00:29:14,170 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 00:29:15,008 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 00:29:16,010 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 00:29:16,577 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:29:16,577 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:29:17,171 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:29:19,087 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 00:29:21,082 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 00:29:21,082 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 00:29:22,247 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:29:26,634 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:29:26,635 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:29:27,248 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:29:31,635 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:29:32,249 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:29:36,089 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 00:29:36,090 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 00:29:38,162 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:29:41,556 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 00:29:41,558 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 00:29:41,558 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 00:29:41,560 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 00:29:41,702 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:29:42,035 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 00:29:42,036 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 00:29:43,560 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:29:46,703 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:29:48,561 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:29:49,092 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 00:29:51,085 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 00:29:51,085 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 00:29:54,223 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:29:56,594 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:29:59,224 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:30:01,595 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:30:04,225 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:30:06,085 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 00:30:06,086 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 00:30:10,210 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:30:11,603 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:30:14,063 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 00:30:14,064 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 00:30:14,065 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 00:30:14,066 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 00:30:14,068 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 00:30:16,068 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:30:16,070 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 00:30:16,605 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:30:19,094 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 00:30:21,085 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 00:30:21,086 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 00:30:21,247 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:30:26,248 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:30:26,610 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:30:31,249 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:30:31,611 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:30:36,085 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 00:30:36,086 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 00:30:36,251 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:30:41,258 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:30:41,619 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:30:43,113 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 00:30:43,114 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 00:30:43,114 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 00:30:43,115 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 00:30:44,098 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 00:30:44,098 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 00:30:46,620 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:30:47,117 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:30:49,098 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 00:30:51,086 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 00:30:51,086 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 00:30:52,193 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:30:56,621 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:30:57,194 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:31:01,622 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:31:02,195 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:31:06,086 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 00:31:06,086 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 00:31:08,184 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:31:11,636 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:31:13,190 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:31:15,424 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 00:31:15,425 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 00:31:15,426 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 00:31:15,427 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 00:31:16,133 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 00:31:16,133 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 00:31:16,824 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:31:16,825 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:31:18,428 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:31:19,102 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 00:31:21,086 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 00:31:21,086 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 00:31:24,174 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:31:26,826 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:31:26,826 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:31:29,175 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:31:31,827 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:31:31,828 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:31:34,175 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:31:36,086 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 00:31:36,086 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 00:31:40,151 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:31:41,829 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:31:45,160 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:31:46,617 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 00:31:46,618 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 00:31:46,618 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 00:31:46,619 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 00:31:47,066 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:31:47,066 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:31:47,165 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 00:31:48,166 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 00:31:49,106 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 00:31:51,086 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 00:31:51,086 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 00:31:51,232 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:31:56,233 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:31:57,068 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:31:57,068 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:32:01,234 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:32:02,152 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:32:06,086 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 00:32:06,087 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 00:32:07,196 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:32:11,313 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:32:12,201 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:32:14,529 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 00:32:14,531 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 00:32:14,531 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 00:32:14,532 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 00:32:15,188 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 00:32:16,189 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 00:32:16,311 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:32:17,533 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:32:19,109 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 00:32:21,086 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 00:32:21,087 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 00:32:23,199 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:32:26,312 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:32:28,200 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:32:31,313 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:32:33,201 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:32:36,086 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 00:32:36,087 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 00:32:38,211 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:32:41,320 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:32:43,218 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:32:46,271 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 00:32:46,273 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 00:32:46,274 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 00:32:46,275 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 00:32:46,666 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:32:46,667 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:32:47,220 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 00:32:48,221 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 00:32:48,276 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:32:49,114 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 00:32:51,087 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 00:32:51,087 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 00:32:54,147 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:32:56,668 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:32:59,148 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:33:01,669 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:33:04,149 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:33:06,086 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 00:33:06,087 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 00:33:09,248 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:33:11,678 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:33:14,255 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:33:15,598 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 00:33:15,600 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 00:33:15,600 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 00:33:15,602 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 00:33:16,241 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 00:33:16,242 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 00:33:16,744 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:33:16,744 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:33:19,117 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 00:33:20,120 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:33:21,087 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 00:33:21,087 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 00:33:25,248 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:33:26,745 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:33:26,745 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:33:30,249 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:33:31,746 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:33:31,746 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:33:35,250 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:33:36,086 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 00:33:36,087 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 00:33:41,164 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:33:41,752 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:33:46,165 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 00:33:46,166 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 00:33:46,166 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 00:33:46,167 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:33:46,168 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 00:33:46,269 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 00:33:46,752 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:33:48,271 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 00:33:49,121 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 00:33:51,087 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 00:33:51,087 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 00:33:51,188 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:33:56,188 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:33:56,753 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:34:01,189 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:34:01,754 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:34:06,087 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 00:34:06,088 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 00:34:06,204 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:34:11,211 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:34:11,764 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:34:15,642 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 00:34:15,643 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 00:34:15,644 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 00:34:15,646 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 00:34:16,300 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 00:34:16,300 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 00:34:16,647 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:34:16,789 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:34:19,125 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 00:34:21,087 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 00:34:21,088 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 00:34:22,193 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:34:26,790 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:34:26,790 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:34:27,194 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:34:31,866 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:34:31,867 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:34:32,194 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:34:36,087 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 00:34:36,088 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 00:34:38,193 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:34:41,868 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:34:41,868 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:34:43,197 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:34:46,875 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:34:47,881 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 00:34:47,883 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 00:34:47,883 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 00:34:47,885 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 00:34:48,328 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 00:34:48,329 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 00:34:48,886 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:34:49,129 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 00:34:51,087 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 00:34:51,088 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 00:34:54,165 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:34:56,880 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:34:59,165 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:35:01,881 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:35:04,166 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:35:06,088 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 00:35:06,088 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 00:35:09,239 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:35:11,892 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:35:14,249 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:35:16,889 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:35:17,497 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 00:35:17,498 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 00:35:17,498 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 00:35:17,500 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 00:35:18,361 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 00:35:18,361 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 00:35:19,134 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 00:35:20,136 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:35:21,088 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 00:35:21,088 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 00:35:25,251 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:35:26,934 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:35:26,934 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:35:30,251 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:35:31,935 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:35:31,936 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:35:35,252 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:35:36,088 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 00:35:36,088 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 00:35:41,243 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:35:41,937 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:35:41,937 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:35:46,251 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:35:46,946 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:35:48,494 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 00:35:48,496 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 00:35:48,497 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 00:35:48,500 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 00:35:49,138 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 00:35:49,392 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 00:35:51,088 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 00:35:51,088 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 00:35:51,394 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 00:35:52,202 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:35:56,945 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:35:57,203 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:36:01,946 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:36:02,203 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:36:06,088 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 00:36:06,089 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 00:36:07,216 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:36:11,997 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:36:12,217 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:36:17,006 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:36:17,222 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:36:19,138 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 00:36:20,347 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 00:36:20,349 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 00:36:20,350 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 00:36:20,352 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 00:36:20,419 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 00:36:21,088 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 00:36:21,089 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 00:36:21,420 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 00:36:22,231 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:36:27,005 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:36:27,232 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:36:32,005 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:36:32,232 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:36:36,088 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 00:36:36,089 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 00:36:38,217 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:36:42,007 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:36:43,218 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:36:47,016 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:36:47,016 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:36:48,226 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:36:49,146 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 00:36:51,090 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 00:36:51,090 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 00:36:51,617 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 00:36:51,619 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 00:36:51,619 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 00:36:51,620 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 00:36:52,448 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 00:36:53,449 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 00:36:53,621 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:36:57,023 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:36:58,621 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:37:02,023 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:37:03,622 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:37:06,090 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 00:37:06,091 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 00:37:09,196 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:37:12,036 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:37:14,197 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:37:17,045 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:37:19,148 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 00:37:20,150 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:37:21,090 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 00:37:21,090 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 00:37:21,537 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 00:37:21,538 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 00:37:21,538 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 00:37:21,540 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 00:37:22,477 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 00:37:23,478 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 00:37:25,541 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:37:27,043 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:37:30,541 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:37:32,044 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:37:35,542 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:37:36,090 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 00:37:36,091 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 00:37:41,165 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:37:41,229 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:37:46,171 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:37:46,234 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:37:49,156 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 00:37:50,520 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 00:37:50,521 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 00:37:50,522 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 00:37:50,523 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 00:37:51,090 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 00:37:51,091 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 00:37:51,507 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 00:37:52,166 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:37:53,509 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 00:37:56,176 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:37:57,166 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:38:01,177 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:38:02,167 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:38:06,091 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 00:38:06,091 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 00:38:07,219 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:38:11,179 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:38:12,219 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:38:16,185 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:38:17,230 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:38:19,162 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 00:38:20,791 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 00:38:20,792 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 00:38:20,792 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 00:38:20,793 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 00:38:21,197 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:38:21,198 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 00:38:21,199 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 00:38:21,533 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 00:38:22,350 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:38:23,535 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 00:38:27,198 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:38:27,198 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:38:27,351 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:38:32,199 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:38:32,199 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:38:32,351 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:38:36,197 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 00:38:36,198 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 00:38:38,261 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:38:41,200 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:38:41,201 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:38:43,262 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:38:46,201 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:38:46,201 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:38:48,268 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:38:49,160 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 00:38:51,198 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 00:38:51,198 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 00:38:51,950 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 00:38:51,952 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 00:38:51,952 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 00:38:51,957 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 00:38:52,559 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 00:38:53,560 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 00:38:53,958 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:38:56,354 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:38:56,354 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:38:58,958 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:39:01,354 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:39:01,355 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:39:03,959 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:39:06,198 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 00:39:06,199 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 00:39:09,272 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:39:11,356 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:39:11,357 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:39:14,272 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:39:16,357 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:39:16,357 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:39:19,163 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 00:39:20,166 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:39:21,208 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 00:39:21,209 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 00:39:23,073 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 00:39:23,075 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 00:39:23,075 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 00:39:23,080 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 00:39:23,583 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 00:39:25,585 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 00:39:26,081 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:39:26,515 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:39:26,516 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:39:31,082 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:39:31,517 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:39:36,083 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:39:36,203 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 00:39:36,203 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 00:39:41,338 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:39:41,584 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:39:46,346 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:39:46,588 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:39:49,166 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 00:39:50,033 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 00:39:50,034 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 00:39:50,035 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 00:39:50,036 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 00:39:50,611 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 00:39:51,203 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 00:39:51,204 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 00:39:51,611 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 00:39:52,339 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:39:56,590 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:39:57,340 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:40:01,591 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:40:02,340 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:40:06,203 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 00:40:06,204 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 00:40:08,272 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:40:11,592 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:40:13,273 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:40:16,601 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:40:18,281 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:40:19,167 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 00:40:20,310 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 00:40:20,311 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 00:40:20,312 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 00:40:20,313 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 00:40:20,636 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 00:40:21,203 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 00:40:21,204 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 00:40:21,637 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 00:40:23,303 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:40:26,706 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:40:28,303 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:40:31,706 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:40:33,304 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:40:36,204 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 00:40:36,204 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 00:40:38,309 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:40:41,815 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:40:43,309 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:40:46,815 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:40:46,816 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:40:48,315 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:40:49,182 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 00:40:51,207 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 00:40:51,208 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 00:40:53,328 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:40:53,408 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 00:40:53,409 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 00:40:53,410 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 00:40:53,412 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 00:40:53,664 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 00:40:55,666 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 00:40:56,825 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:40:58,413 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:41:01,826 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:41:03,413 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:41:06,207 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 00:41:06,208 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 00:41:09,305 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:41:11,828 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:41:14,306 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:41:16,837 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:41:19,178 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 00:41:20,181 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:41:21,211 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 00:41:21,211 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 00:41:22,719 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 00:41:22,721 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 00:41:22,721 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 00:41:22,724 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 00:41:23,696 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 00:41:23,697 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 00:41:25,725 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:41:26,864 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:41:30,725 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:41:31,865 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:41:35,726 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:41:36,211 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 00:41:36,212 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 00:41:41,274 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:41:41,867 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:41:41,867 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:41:46,275 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:41:46,868 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:41:46,868 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:41:49,187 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 00:41:51,217 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 00:41:51,218 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 00:41:51,358 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:41:53,504 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 00:41:53,505 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 00:41:53,505 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 00:41:53,510 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 00:41:53,730 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 00:41:55,732 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 00:41:56,511 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:41:56,903 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:42:01,512 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:42:01,903 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:42:01,904 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:42:06,213 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 00:42:06,213 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 00:42:07,286 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:42:11,905 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:42:11,906 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:42:12,286 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:42:16,906 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:42:16,906 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:42:17,287 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:42:19,187 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 00:42:21,213 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 00:42:21,214 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 00:42:22,348 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:42:26,917 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:42:27,354 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:42:27,366 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 00:42:27,367 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 00:42:27,368 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 00:42:27,373 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 00:42:27,763 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 00:42:29,764 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 00:42:31,919 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:42:32,375 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:42:36,213 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 00:42:36,214 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 00:42:38,286 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:42:42,022 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:42:42,022 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:42:43,287 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:42:47,022 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:42:47,023 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:42:48,288 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:42:49,191 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 00:42:51,213 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 00:42:51,214 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 00:42:53,291 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:42:57,041 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:42:58,160 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 00:42:58,161 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 00:42:58,161 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 00:42:58,164 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 00:42:58,785 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 00:42:59,165 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:42:59,787 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 00:43:02,035 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:43:04,165 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:43:06,213 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 00:43:06,214 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 00:43:09,377 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:43:12,036 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:43:14,377 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:43:17,037 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:43:19,195 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 00:43:20,197 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:43:21,215 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 00:43:21,216 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 00:43:25,307 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:43:27,040 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 00:43:27,042 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 00:43:27,042 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 00:43:27,043 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:43:27,044 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 00:43:27,817 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 00:43:29,819 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 00:43:31,046 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:43:32,044 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:43:36,046 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:43:36,215 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 00:43:36,216 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 00:43:41,313 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:43:42,046 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:43:46,314 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:43:47,046 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:43:49,199 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 00:43:51,225 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 00:43:51,227 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 00:43:51,346 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:43:56,305 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 00:43:56,306 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 00:43:56,306 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 00:43:56,307 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 00:43:56,839 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 00:43:57,051 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:43:57,308 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:43:57,841 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 00:44:02,052 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:44:02,308 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:44:06,221 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 00:44:06,222 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 00:44:07,370 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:44:12,053 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:44:12,371 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:44:17,054 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:44:17,371 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:44:19,203 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 00:44:21,221 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 00:44:21,221 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 00:44:23,312 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:44:26,254 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 00:44:26,256 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 00:44:26,256 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 00:44:26,258 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 00:44:26,870 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 00:44:27,059 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:44:27,871 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 00:44:29,258 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:44:32,060 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:44:34,259 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:44:36,221 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 00:44:36,221 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 00:44:39,338 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:44:42,061 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:44:44,339 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:44:47,062 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:44:49,207 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 00:44:50,209 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:44:51,221 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 00:44:51,222 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 00:44:55,398 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:44:57,071 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:44:59,173 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 00:44:59,175 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 00:44:59,175 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 00:44:59,177 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 00:44:59,898 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 00:45:01,178 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:45:01,900 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 00:45:02,072 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:45:06,178 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:45:06,221 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 00:45:06,222 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 00:45:11,319 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:45:11,384 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:45:16,326 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:45:16,385 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:45:19,211 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 00:45:21,222 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 00:45:21,222 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 00:45:22,294 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:45:26,334 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:45:27,303 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:45:29,194 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 00:45:29,196 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 00:45:29,197 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 00:45:29,198 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 00:45:29,925 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 00:45:31,343 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:45:31,927 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 00:45:33,199 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:45:36,222 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 00:45:36,222 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 00:45:38,386 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:45:41,344 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:45:43,387 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:45:46,345 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:45:48,387 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:45:49,215 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 00:45:51,222 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 00:45:51,222 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 00:45:54,302 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:45:56,352 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:45:59,309 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:46:00,539 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 00:46:00,541 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 00:46:00,542 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 00:46:00,543 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 00:46:00,962 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 00:46:01,355 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:46:01,963 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 00:46:04,545 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:46:06,222 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 00:46:06,223 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 00:46:10,361 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:46:11,357 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:46:15,361 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:46:16,358 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:46:19,220 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 00:46:21,222 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 00:46:21,222 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:46:21,223 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 00:46:26,311 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:46:26,360 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:46:31,319 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:46:31,372 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:46:32,693 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 00:46:32,695 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 00:46:32,696 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 00:46:32,697 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 00:46:32,997 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 00:46:33,998 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 00:46:36,222 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 00:46:36,223 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 00:46:37,304 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:46:41,370 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:46:42,305 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:46:46,371 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:46:47,306 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:46:49,223 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 00:46:51,222 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 00:46:51,223 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 00:46:52,376 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:46:56,372 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:46:57,376 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:47:01,380 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:47:02,383 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:47:05,464 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 00:47:05,466 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 00:47:05,467 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 00:47:05,468 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 00:47:06,024 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 00:47:06,222 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 00:47:06,223 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 00:47:08,026 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 00:47:08,290 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:47:11,387 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:47:13,290 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:47:16,388 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:47:18,291 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:47:19,227 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 00:47:21,222 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 00:47:21,223 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 00:47:23,303 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:47:26,389 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:47:28,303 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:47:31,401 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:47:33,311 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:47:34,705 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 00:47:34,707 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 00:47:34,707 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 00:47:34,709 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 00:47:35,055 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 00:47:36,056 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 00:47:36,223 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 00:47:36,223 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 00:47:38,385 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:47:41,400 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:47:43,386 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:47:46,401 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:47:48,386 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:47:49,231 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 00:47:51,223 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 00:47:51,223 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 00:47:54,371 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:47:56,402 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:47:59,378 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:48:01,407 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:48:04,375 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:48:04,495 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 00:48:04,497 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 00:48:04,498 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 00:48:04,499 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 00:48:05,081 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 00:48:06,082 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 00:48:06,223 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 00:48:06,224 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 00:48:10,336 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:48:11,412 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:48:15,336 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:48:16,413 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:48:19,235 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 00:48:21,223 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 00:48:21,224 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 00:48:21,337 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:48:26,337 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:48:26,414 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:48:31,338 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:48:31,415 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:48:36,227 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 00:48:36,227 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 00:48:36,359 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:48:37,247 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 00:48:37,249 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 00:48:37,250 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 00:48:37,251 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 00:48:38,109 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 00:48:38,110 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 00:48:41,657 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:48:42,252 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:48:46,658 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:48:47,253 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:48:49,240 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 00:48:51,227 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 00:48:51,228 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 00:48:52,391 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:48:56,227 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:48:57,391 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:49:01,228 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:49:02,392 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:49:06,233 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 00:49:06,234 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 00:49:08,297 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 00:49:08,299 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 00:49:08,300 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 00:49:08,300 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:49:08,301 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 00:49:09,142 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 00:49:10,144 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 00:49:11,245 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:49:13,303 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:49:16,246 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:49:18,303 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:49:19,243 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 00:49:21,379 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:49:21,380 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 00:49:21,381 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 00:49:23,448 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:49:26,698 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:49:28,449 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:49:31,699 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:49:33,455 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:49:36,382 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 00:49:36,383 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 00:49:36,784 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 00:49:36,786 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 00:49:36,787 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 00:49:36,788 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 00:49:37,167 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 00:49:38,169 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 00:49:38,789 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:49:41,387 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:49:43,790 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:49:46,387 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:49:48,791 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:49:49,247 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 00:49:51,381 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 00:49:51,382 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 00:49:54,542 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:49:56,432 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:49:59,543 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:50:01,442 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:50:04,553 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:50:05,884 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 00:50:05,885 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 00:50:05,886 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 00:50:05,887 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 00:50:06,191 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 00:50:06,381 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 00:50:06,382 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 00:50:08,193 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 00:50:10,528 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:50:11,441 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:50:15,529 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:50:16,442 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:50:19,251 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 00:50:21,254 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:50:21,381 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 00:50:21,382 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 00:50:26,443 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:50:26,463 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:50:31,444 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:50:31,464 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:50:36,389 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 00:50:36,390 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 00:50:36,519 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:50:37,762 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 00:50:37,763 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 00:50:37,763 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 00:50:37,765 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 00:50:38,222 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 00:50:40,223 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 00:50:41,449 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:50:41,765 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:50:46,449 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:50:46,766 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:50:49,255 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 00:50:51,384 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 00:50:51,385 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 00:50:52,473 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:50:56,451 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:50:57,474 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:51:01,451 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:51:02,481 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:51:06,395 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 00:51:06,396 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 00:51:07,502 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 00:51:07,504 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 00:51:07,505 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 00:51:07,505 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:51:07,506 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 00:51:08,251 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 00:51:08,251 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 00:51:11,456 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:51:12,508 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:51:16,456 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:51:17,509 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:51:19,259 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 00:51:21,390 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 00:51:21,391 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 00:51:23,481 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:51:26,458 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:51:28,481 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:51:31,468 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:51:33,491 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:51:35,129 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 00:51:35,131 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 00:51:35,131 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 00:51:35,133 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 00:51:35,288 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 00:51:36,290 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 00:51:36,390 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 00:51:36,391 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 00:51:38,532 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:51:41,538 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:51:43,533 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:51:46,539 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:51:48,534 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:51:49,264 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 00:51:51,390 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 00:51:51,391 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 00:51:54,477 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:51:56,541 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:51:59,478 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:52:01,550 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:52:04,486 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:52:06,094 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 00:52:06,095 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 00:52:06,096 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 00:52:06,097 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 00:52:06,325 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 00:52:06,495 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 00:52:06,495 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 00:52:08,326 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 00:52:09,609 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:52:11,553 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:52:14,610 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:52:16,554 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:52:19,266 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 00:52:20,268 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:52:21,493 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 00:52:21,494 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 00:52:25,657 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:52:26,555 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:52:30,658 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:52:31,560 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:52:35,665 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:52:36,090 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 00:52:36,091 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 00:52:36,092 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 00:52:36,093 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 00:52:36,352 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 00:52:36,497 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 00:52:36,498 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 00:52:38,354 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 00:52:41,566 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:52:41,592 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:52:46,567 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:52:46,592 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:52:49,270 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 00:52:51,496 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 00:52:51,497 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 00:52:51,613 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:52:56,580 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:52:56,613 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:53:01,581 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:53:01,614 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:53:06,507 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 00:53:06,508 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 00:53:06,679 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:53:07,945 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 00:53:07,946 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 00:53:07,947 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 00:53:07,948 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 00:53:08,382 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 00:53:10,383 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 00:53:11,600 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:53:11,949 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:53:16,601 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:53:16,950 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:53:19,274 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 00:53:21,500 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 00:53:21,501 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 00:53:22,589 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:53:26,603 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:53:27,590 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:53:31,603 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:53:32,591 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:53:36,506 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 00:53:36,509 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 00:53:37,617 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:53:39,898 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 00:53:39,900 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 00:53:39,901 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 00:53:39,902 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 00:53:40,414 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 00:53:41,608 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:53:42,415 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 00:53:42,903 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:53:46,609 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:53:47,904 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:53:49,278 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 00:53:51,507 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 00:53:51,507 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 00:53:53,654 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:53:56,611 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:53:58,654 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:54:01,611 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:54:03,655 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:54:06,511 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 00:54:06,516 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 00:54:08,655 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:54:11,311 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 00:54:11,312 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 00:54:11,313 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 00:54:11,313 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 00:54:11,443 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 00:54:12,409 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:54:12,410 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:54:12,445 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 00:54:14,314 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:54:17,410 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:54:17,410 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:54:19,282 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 00:54:20,285 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:54:21,508 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 00:54:21,509 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 00:54:25,675 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:54:27,412 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:54:27,412 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:54:30,676 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:54:32,413 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:54:32,413 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:54:35,686 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:54:36,517 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 00:54:36,519 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 00:54:41,235 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 00:54:41,237 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 00:54:41,237 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 00:54:41,238 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:54:41,239 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 00:54:41,473 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 00:54:42,149 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:54:42,150 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:54:42,474 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 00:54:46,240 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:54:47,151 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:54:47,151 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:54:49,286 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 00:54:51,289 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:54:51,510 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 00:54:51,510 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 00:54:56,594 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:54:57,152 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:55:01,594 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:55:02,153 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:55:06,521 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 00:55:06,525 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 00:55:06,618 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:55:11,619 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:55:12,159 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:55:12,159 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:55:12,298 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 00:55:12,299 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 00:55:12,299 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 00:55:12,300 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 00:55:12,511 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 00:55:14,513 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 00:55:17,160 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:55:17,160 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:55:17,302 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:55:19,290 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 00:55:21,511 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 00:55:21,512 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 00:55:22,673 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:55:27,161 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:55:27,674 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:55:32,162 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:55:32,674 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:55:36,511 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 00:55:36,512 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 00:55:38,655 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:55:42,170 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:55:43,127 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 00:55:43,128 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 00:55:43,129 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 00:55:43,130 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 00:55:43,542 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 00:55:44,131 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:55:44,543 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 00:55:47,275 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:55:49,131 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:55:49,294 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 00:55:51,511 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 00:55:51,512 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 00:55:54,679 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:55:57,276 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:55:59,680 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:56:02,277 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:56:04,680 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:56:06,511 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 00:56:06,512 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 00:56:10,579 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:56:12,287 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:56:13,576 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 00:56:13,578 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 00:56:13,578 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 00:56:13,580 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 00:56:14,572 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 00:56:14,572 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 00:56:15,580 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:56:17,286 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:56:19,298 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 00:56:21,300 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:56:21,512 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 00:56:21,512 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 00:56:26,595 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:56:27,287 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:56:31,596 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:56:32,288 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:56:36,512 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 00:56:36,512 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 00:56:36,669 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:56:41,678 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:56:42,300 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:56:42,649 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 00:56:42,650 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 00:56:42,650 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 00:56:42,651 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 00:56:43,600 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 00:56:44,601 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 00:56:47,299 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:56:47,652 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:56:49,302 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 00:56:51,512 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 00:56:51,512 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 00:56:53,638 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:56:57,300 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:56:58,639 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:57:02,301 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:57:03,639 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:57:06,512 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 00:57:06,512 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 00:57:08,646 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:57:12,315 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:57:13,651 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:57:15,971 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 00:57:15,972 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 00:57:15,972 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 00:57:15,973 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 00:57:16,631 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 00:57:16,631 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 00:57:17,372 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:57:17,372 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:57:18,974 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:57:19,306 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 00:57:21,512 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 00:57:21,513 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 00:57:24,576 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:57:27,373 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:57:27,374 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:57:29,576 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:57:32,374 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:57:32,375 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:57:34,577 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:57:36,512 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 00:57:36,513 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 00:57:39,593 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:57:42,380 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:57:44,604 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:57:46,769 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 00:57:46,770 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 00:57:46,771 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 00:57:46,772 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 00:57:47,381 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:57:47,660 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 00:57:48,661 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 00:57:49,310 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 00:57:50,312 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:57:51,512 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 00:57:51,513 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 00:57:55,597 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:57:57,383 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:58:00,598 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:58:02,383 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:58:05,599 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:58:06,513 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 00:58:06,513 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 00:58:10,664 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:58:12,385 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:58:15,669 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:58:17,394 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:58:19,314 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 00:58:19,411 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 00:58:19,413 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 00:58:19,413 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 00:58:19,415 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 00:58:19,688 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 00:58:20,689 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 00:58:21,416 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:58:21,512 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 00:58:21,513 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 00:58:26,677 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:58:27,395 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:58:31,678 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:58:32,396 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:58:32,396 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:58:36,513 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 00:58:36,513 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 00:58:37,653 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:58:42,398 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:58:42,398 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:58:42,654 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:58:47,411 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:58:47,663 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:58:49,317 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 00:58:50,984 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 00:58:50,985 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 00:58:50,985 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 00:58:50,986 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 00:58:51,513 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 00:58:51,514 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 00:58:51,715 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 00:58:52,716 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 00:58:53,598 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:58:57,404 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:58:58,598 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:59:02,405 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:59:03,599 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:59:06,513 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 00:59:06,514 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 00:59:08,615 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:59:12,415 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:59:12,415 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:59:13,616 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:59:17,428 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:59:18,620 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:59:19,325 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 00:59:21,519 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 00:59:21,520 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 00:59:23,080 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 00:59:23,081 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 00:59:23,082 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 00:59:23,084 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 00:59:23,739 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 00:59:24,085 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:59:25,741 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 00:59:27,483 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:59:27,483 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:59:29,085 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:59:32,483 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:59:32,484 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:59:34,086 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:59:36,517 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 00:59:36,518 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 00:59:39,670 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:59:42,485 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:59:42,486 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:59:44,670 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:59:47,486 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:59:47,487 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 00:59:49,327 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 00:59:50,332 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:59:51,520 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 00:59:51,523 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 00:59:53,540 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 00:59:53,542 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 00:59:53,542 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 00:59:53,545 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 00:59:53,763 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 00:59:55,546 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 00:59:55,765 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 00:59:57,493 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:00:00,547 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:00:02,493 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:00:05,547 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:00:06,518 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 01:00:06,519 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 01:00:10,615 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:00:12,495 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:00:15,616 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:00:17,496 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:00:19,326 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 01:00:21,327 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:00:21,520 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 01:00:21,520 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 01:00:24,165 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 01:00:24,166 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 01:00:24,166 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 01:00:24,171 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 01:00:24,790 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 01:00:25,791 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 01:00:26,584 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:00:26,584 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:00:27,172 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:00:31,585 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:00:31,585 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:00:32,173 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:00:36,520 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 01:00:36,521 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 01:00:37,682 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:00:41,669 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:00:41,669 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:00:42,683 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:00:46,669 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:00:46,670 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:00:47,690 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:00:49,329 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 01:00:51,526 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 01:00:51,528 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 01:00:53,018 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 01:00:53,020 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 01:00:53,020 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 01:00:53,020 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:00:53,022 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 01:00:53,822 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 01:00:55,823 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 01:00:56,676 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:00:56,676 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:00:58,023 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:01:01,677 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:01:03,023 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:01:06,524 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 01:01:06,525 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 01:01:08,622 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:01:11,679 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:01:11,679 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:01:13,623 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:01:16,680 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:01:18,629 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:01:19,334 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 01:01:21,530 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 01:01:21,531 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 01:01:23,659 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:01:24,030 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 01:01:24,031 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 01:01:24,031 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 01:01:24,032 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 01:01:24,859 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 01:01:25,860 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 01:01:26,689 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:01:29,033 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:01:31,690 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:01:34,034 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:01:36,528 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 01:01:36,529 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 01:01:39,680 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:01:41,692 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:01:44,681 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:01:46,693 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:01:49,349 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 01:01:50,350 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:01:51,533 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 01:01:51,537 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 01:01:52,895 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 01:01:52,897 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 01:01:52,897 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 01:01:52,899 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 01:01:53,886 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 01:01:55,888 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 01:01:55,900 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:01:56,701 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:02:00,900 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:02:01,701 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:02:05,901 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:02:06,530 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 01:02:06,531 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 01:02:11,662 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:02:11,703 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:02:16,670 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:02:16,706 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:02:19,346 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 01:02:21,530 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 01:02:21,531 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 01:02:21,630 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 01:02:21,632 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 01:02:21,632 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 01:02:21,634 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 01:02:21,920 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 01:02:22,635 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:02:23,922 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 01:02:26,781 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:02:27,636 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:02:31,782 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:02:31,782 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:02:32,636 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:02:36,530 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 01:02:36,531 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 01:02:37,678 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:02:41,784 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:02:42,678 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:02:46,799 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:02:47,683 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:02:49,345 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 01:02:50,497 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 01:02:50,499 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 01:02:50,499 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 01:02:50,501 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 01:02:50,948 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 01:02:51,531 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 01:02:51,531 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 01:02:51,949 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 01:02:52,689 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:02:56,903 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:02:57,690 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:03:01,904 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:03:02,691 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:03:06,530 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 01:03:06,531 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 01:03:07,693 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:03:11,915 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:03:12,696 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:03:16,914 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:03:17,703 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:03:17,710 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 01:03:17,711 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 01:03:17,712 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 01:03:17,713 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 01:03:17,982 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 01:03:19,348 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 01:03:19,984 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 01:03:21,531 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 01:03:21,531 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 01:03:23,674 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:03:27,139 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:03:27,140 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:03:28,674 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:03:32,140 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:03:32,141 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:03:33,675 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:03:36,531 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 01:03:36,531 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 01:03:39,618 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:03:42,147 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:03:44,629 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:03:47,148 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:03:47,426 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 01:03:47,427 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 01:03:47,428 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 01:03:47,429 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 01:03:48,010 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 01:03:49,352 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 01:03:50,011 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 01:03:50,355 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:03:51,531 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 01:03:51,532 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 01:03:55,682 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:03:57,150 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:04:00,683 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:04:02,151 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:04:05,683 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:04:06,531 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 01:04:06,532 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 01:04:11,600 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:04:12,156 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:04:16,608 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:04:17,158 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:04:17,483 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 01:04:17,484 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 01:04:17,485 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 01:04:17,486 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 01:04:18,037 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 01:04:19,356 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 01:04:20,038 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 01:04:21,531 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 01:04:21,532 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 01:04:21,617 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:04:26,617 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:04:27,160 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:04:31,618 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:04:32,161 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:04:36,531 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 01:04:36,532 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 01:04:36,667 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:04:41,668 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:04:42,163 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:04:46,681 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:04:47,172 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:04:48,857 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 01:04:48,859 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 01:04:48,859 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 01:04:48,861 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 01:04:49,065 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 01:04:49,360 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 01:04:51,531 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 01:04:51,532 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 01:04:52,067 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 01:04:52,651 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:04:57,257 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:04:57,257 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:04:57,652 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:05:02,300 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:05:02,301 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:05:02,653 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:05:06,532 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 01:05:06,532 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 01:05:08,616 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:05:12,302 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:05:12,302 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:05:13,617 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:05:17,314 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:05:18,623 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:05:19,367 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 01:05:19,595 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 01:05:19,597 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 01:05:19,598 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 01:05:19,599 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 01:05:20,092 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 01:05:21,537 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 01:05:21,538 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 01:05:22,093 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 01:05:23,625 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:05:27,311 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:05:28,626 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:05:32,312 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:05:33,626 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:05:36,537 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 01:05:36,538 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 01:05:38,650 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:05:42,314 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:05:42,314 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:05:43,652 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:05:47,328 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:05:48,657 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:05:48,903 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 01:05:48,905 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 01:05:48,905 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 01:05:48,907 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 01:05:49,126 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 01:05:49,369 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 01:05:50,127 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 01:05:51,537 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 01:05:51,538 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 01:05:53,678 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:05:57,320 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:05:58,679 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:06:02,321 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:06:03,679 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:06:06,537 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 01:06:06,538 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 01:06:08,685 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:06:12,322 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:06:13,692 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:06:17,328 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:06:18,695 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:06:19,372 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 01:06:19,440 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 01:06:19,441 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 01:06:19,442 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 01:06:19,443 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 01:06:20,158 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 01:06:21,537 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 01:06:21,538 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 01:06:22,160 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 01:06:24,603 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:06:27,331 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:06:29,603 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:06:32,332 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:06:34,604 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:06:36,537 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 01:06:36,538 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 01:06:39,617 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:06:42,334 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:06:44,624 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:06:47,341 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:06:49,039 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 01:06:49,041 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 01:06:49,041 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 01:06:49,045 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 01:06:49,188 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 01:06:49,377 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 01:06:50,189 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 01:06:50,379 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:06:51,538 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 01:06:51,538 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 01:06:55,646 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:06:57,443 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:06:57,444 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:07:00,647 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:07:02,445 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:07:02,445 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:07:05,647 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:07:06,538 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 01:07:06,538 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 01:07:10,658 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:07:12,446 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:07:12,447 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:07:15,664 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:07:17,453 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:07:19,382 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 01:07:19,866 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 01:07:19,868 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 01:07:19,869 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 01:07:19,870 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 01:07:20,220 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 01:07:20,870 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:07:21,538 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 01:07:21,538 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 01:07:22,222 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 01:07:26,614 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:07:27,457 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:07:31,614 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:07:32,458 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:07:36,538 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 01:07:36,539 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 01:07:36,677 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:07:41,677 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:07:42,459 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:07:46,681 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:07:47,470 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:07:49,388 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 01:07:51,508 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 01:07:51,510 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 01:07:51,510 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 01:07:51,512 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 01:07:51,913 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 01:07:51,914 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 01:07:52,022 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:07:52,253 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 01:07:52,253 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 01:07:57,022 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:07:57,471 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:08:02,023 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:08:02,472 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:08:06,653 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 01:08:06,653 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 01:08:07,762 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:08:12,473 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:08:12,473 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:08:12,763 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:08:17,481 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:08:17,772 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:08:19,303 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 01:08:19,305 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 01:08:19,305 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 01:08:19,307 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 01:08:19,392 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 01:08:20,275 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 01:08:20,275 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 01:08:21,653 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 01:08:21,653 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 01:08:23,719 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:08:26,710 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:08:28,719 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:08:31,711 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:08:33,720 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:08:36,653 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 01:08:36,653 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 01:08:38,807 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:08:41,777 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:08:43,807 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:08:46,783 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:08:48,814 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:08:49,393 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 01:08:51,653 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 01:08:51,653 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 01:08:51,718 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 01:08:51,814 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 01:08:51,815 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 01:08:51,817 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 01:08:52,299 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 01:08:53,818 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:08:54,300 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 01:08:56,865 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:08:58,818 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:09:01,865 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:09:03,819 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:09:06,653 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 01:09:06,654 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 01:09:09,767 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:09:11,867 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:09:14,768 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:09:16,875 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:09:19,404 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 01:09:20,405 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:09:21,507 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 01:09:21,508 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 01:09:21,509 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 01:09:21,510 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 01:09:21,913 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:09:21,917 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 01:09:21,917 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 01:09:22,324 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 01:09:22,325 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 01:09:26,028 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:09:27,168 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:09:31,029 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:09:32,169 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:09:36,029 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:09:36,914 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 01:09:36,915 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 01:09:41,081 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:09:41,917 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:09:46,085 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:09:46,918 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:09:49,405 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 01:09:50,136 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 01:09:50,137 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 01:09:50,138 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 01:09:50,142 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 01:09:50,348 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 01:09:51,143 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:09:51,914 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 01:09:51,915 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 01:09:52,350 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 01:09:56,922 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:09:57,075 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:10:01,923 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:10:02,076 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:10:06,914 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 01:10:06,915 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 01:10:08,002 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:10:12,049 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:10:13,002 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:10:17,060 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:10:18,009 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:10:19,413 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 01:10:20,526 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 01:10:20,529 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 01:10:20,529 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 01:10:20,533 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 01:10:21,379 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 01:10:21,914 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 01:10:21,915 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 01:10:22,381 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 01:10:23,062 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:10:27,063 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:10:28,062 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:10:32,064 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:10:33,063 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:10:36,914 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 01:10:36,915 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 01:10:39,045 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:10:42,066 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:10:44,046 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:10:47,072 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:10:49,057 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:10:49,416 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 01:10:51,914 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 01:10:51,915 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 01:10:51,972 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 01:10:52,058 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 01:10:52,059 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 01:10:52,064 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 01:10:52,411 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 01:10:54,066 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:10:54,413 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 01:10:57,378 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:10:57,379 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:10:59,066 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:11:02,379 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:11:04,067 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:11:06,915 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 01:11:06,915 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 01:11:09,076 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:11:12,380 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:11:14,076 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:11:17,381 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:11:19,084 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:11:19,421 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 01:11:21,921 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 01:11:21,922 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 01:11:24,618 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 01:11:24,619 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 01:11:24,620 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 01:11:24,620 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:11:24,623 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 01:11:25,441 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 01:11:26,442 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 01:11:27,387 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:11:29,624 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:11:32,388 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:11:34,624 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:11:36,918 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 01:11:36,919 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 01:11:39,994 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:11:42,390 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:11:44,994 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:11:47,390 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:11:49,431 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 01:11:50,431 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:11:51,926 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 01:11:51,927 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 01:11:54,766 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 01:11:54,768 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 01:11:54,768 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 01:11:54,773 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 01:11:55,468 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 01:11:55,774 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:11:56,468 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 01:11:57,400 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:12:00,774 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:12:02,401 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:12:05,775 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:12:07,148 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 01:12:07,149 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 01:12:11,286 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:12:12,402 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:12:16,286 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:12:17,403 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:12:19,431 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 01:12:21,436 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:12:22,061 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 01:12:22,062 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 01:12:25,203 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 01:12:25,205 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 01:12:25,206 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 01:12:25,210 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 01:12:25,496 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 01:12:26,497 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 01:12:27,211 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:12:27,615 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:12:27,616 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:12:32,211 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:12:32,616 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:12:32,617 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:12:37,058 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 01:12:37,058 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 01:12:38,156 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:12:42,735 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:12:42,735 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:12:43,157 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:12:47,736 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:12:47,737 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:12:48,157 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:12:49,441 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 01:12:52,066 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 01:12:52,071 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 01:12:53,159 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:12:55,073 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 01:12:55,074 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 01:12:55,074 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 01:12:55,076 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 01:12:55,534 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 01:12:56,536 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 01:12:57,747 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:12:57,748 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:12:59,077 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:13:02,748 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:13:04,077 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:13:07,296 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 01:13:07,297 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 01:13:09,408 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:13:12,750 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:13:12,750 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:13:14,409 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:13:17,750 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:13:17,751 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:13:19,416 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:13:19,438 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 01:13:22,207 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 01:13:22,208 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 01:13:25,290 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 01:13:25,292 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 01:13:25,293 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 01:13:25,293 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:13:25,295 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 01:13:25,560 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 01:13:26,561 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 01:13:27,758 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:13:30,296 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:13:32,760 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:13:35,297 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:13:37,203 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 01:13:37,204 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 01:13:40,298 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:13:42,843 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:13:42,843 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:13:45,298 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:13:47,844 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:13:47,844 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:13:49,438 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 01:13:50,439 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:13:52,216 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 01:13:52,217 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 01:13:54,176 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 01:13:54,178 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 01:13:54,178 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 01:13:54,179 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 01:13:54,586 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 01:13:56,180 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:13:56,588 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 01:13:57,852 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:14:01,181 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:14:02,853 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:14:06,181 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:14:07,209 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 01:14:07,210 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 01:14:11,300 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:14:12,855 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:14:16,301 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:14:17,855 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:14:19,442 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 01:14:21,446 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:14:22,215 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 01:14:22,215 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 01:14:24,610 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 01:14:24,612 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 01:14:24,612 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 01:14:24,614 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 01:14:25,614 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 01:14:26,615 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:14:26,616 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 01:14:28,019 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:14:28,019 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:14:31,616 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:14:33,019 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:14:33,020 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:14:36,616 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:14:37,215 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 01:14:37,216 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 01:14:42,356 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:14:43,021 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:14:43,021 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:14:47,357 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:14:48,022 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:14:48,022 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:14:49,447 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 01:14:52,217 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 01:14:52,217 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 01:14:53,305 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:14:57,226 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 01:14:57,228 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 01:14:57,229 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 01:14:57,234 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 01:14:57,652 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 01:14:58,031 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:14:58,031 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:14:58,653 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 01:14:59,235 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:15:03,032 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:15:03,032 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:15:04,235 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:15:07,217 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 01:15:07,217 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 01:15:09,487 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:15:13,034 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:15:14,488 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:15:18,034 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:15:18,035 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:15:19,449 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 01:15:20,451 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:15:22,219 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 01:15:22,219 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 01:15:26,294 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:15:27,162 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 01:15:27,164 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 01:15:27,164 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 01:15:27,165 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 01:15:27,677 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 01:15:28,043 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:15:28,678 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 01:15:32,166 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:15:33,046 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:15:37,167 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:15:37,218 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 01:15:37,218 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 01:15:42,329 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:15:43,047 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:15:47,329 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:15:48,048 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:15:48,048 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:15:49,454 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 01:15:52,229 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 01:15:52,238 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 01:15:52,378 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:15:57,387 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:15:57,748 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 01:15:57,749 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 01:15:57,749 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 01:15:57,750 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 01:15:58,153 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:15:58,154 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:15:58,699 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 01:15:58,699 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 01:16:02,752 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:16:03,155 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:16:03,155 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:16:07,223 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 01:16:07,224 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 01:16:08,318 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:16:12,303 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:16:12,304 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:16:13,319 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:16:17,305 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:16:17,305 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:16:18,320 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:16:19,457 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 01:16:22,223 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 01:16:22,224 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 01:16:23,333 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:16:27,310 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:16:28,341 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:16:29,760 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 01:16:29,762 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 01:16:29,762 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 01:16:29,768 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 01:16:30,724 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 01:16:30,725 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 01:16:32,310 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:16:33,769 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:16:37,223 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 01:16:37,224 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 01:16:39,354 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:16:42,311 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:16:44,354 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:16:47,312 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:16:49,355 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:16:49,461 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 01:16:52,223 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 01:16:52,224 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 01:16:54,368 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:16:57,313 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:16:59,373 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:17:02,325 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:17:02,471 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 01:17:02,472 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 01:17:02,473 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 01:17:02,476 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 01:17:02,754 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 01:17:04,477 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:17:04,755 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 01:17:07,223 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 01:17:07,224 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 01:17:10,339 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:17:12,323 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:17:15,340 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:17:17,324 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:17:19,465 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 01:17:20,468 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:17:22,224 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 01:17:22,224 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 01:17:26,308 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:17:27,326 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:17:31,315 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:17:32,336 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:17:34,357 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 01:17:34,359 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 01:17:34,359 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 01:17:34,365 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 01:17:34,779 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 01:17:34,780 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 01:17:36,366 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:17:37,224 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 01:17:37,224 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 01:17:42,304 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:17:42,331 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:17:47,304 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:17:47,332 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:17:49,469 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 01:17:52,224 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 01:17:52,224 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 01:17:52,320 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:17:57,320 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:17:57,334 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:18:02,325 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:18:02,351 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:18:03,808 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 01:18:03,810 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 01:18:03,810 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 01:18:03,816 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 01:18:04,812 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 01:18:04,812 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 01:18:07,224 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 01:18:07,224 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 01:18:07,359 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:18:12,348 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:18:12,359 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:18:17,349 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:18:17,349 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:18:17,359 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:18:19,473 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 01:18:22,224 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 01:18:22,224 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 01:18:23,289 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:18:27,350 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:18:28,290 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:18:32,351 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:18:33,297 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:18:35,236 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 01:18:35,238 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 01:18:35,238 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 01:18:35,241 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 01:18:35,843 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 01:18:36,844 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 01:18:37,224 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 01:18:37,225 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 01:18:38,384 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:18:42,384 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:18:43,385 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:18:47,384 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:18:48,385 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:18:49,478 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 01:18:52,224 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 01:18:52,225 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 01:18:54,342 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:18:57,388 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:18:59,350 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:19:02,390 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:19:02,755 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 01:19:02,757 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 01:19:02,757 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 01:19:02,759 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 01:19:02,866 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 01:19:04,760 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:19:04,868 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 01:19:07,224 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 01:19:07,225 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 01:19:10,338 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:19:12,392 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:19:15,339 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:19:17,392 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:19:19,481 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 01:19:20,484 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:19:22,225 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 01:19:22,225 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 01:19:26,299 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:19:27,394 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:19:31,305 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:19:32,402 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:19:34,580 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 01:19:34,582 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 01:19:34,583 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 01:19:34,584 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 01:19:34,910 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 01:19:34,911 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 01:19:36,585 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:19:37,225 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 01:19:37,225 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 01:19:42,373 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:19:42,400 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:19:47,373 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:19:47,401 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:19:49,486 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 01:19:52,225 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 01:19:52,226 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 01:19:53,304 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:19:57,441 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:19:58,305 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:20:02,443 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:20:03,313 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:20:07,234 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 01:20:07,236 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 01:20:08,349 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:20:08,771 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 01:20:08,771 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 01:20:08,772 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 01:20:08,773 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 01:20:08,950 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 01:20:10,952 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 01:20:12,450 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:20:13,774 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:20:17,451 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:20:18,775 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:20:19,490 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 01:20:22,227 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 01:20:22,228 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 01:20:24,332 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:20:27,452 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:20:29,333 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:20:32,453 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:20:34,333 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:20:37,236 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 01:20:37,237 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 01:20:39,358 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:20:41,786 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 01:20:41,788 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 01:20:41,788 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 01:20:41,790 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 01:20:41,977 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 01:20:42,862 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:20:42,978 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 01:20:44,791 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:20:47,863 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:20:49,494 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 01:20:50,496 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:20:52,227 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 01:20:52,228 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 01:20:56,374 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:20:57,864 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:21:01,374 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:21:02,865 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:21:06,375 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:21:07,238 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 01:21:07,246 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 01:21:11,422 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:21:12,877 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:21:13,288 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 01:21:13,289 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 01:21:13,289 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 01:21:13,291 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 01:21:14,006 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 01:21:15,008 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 01:21:17,292 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:21:17,878 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:21:19,498 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 01:21:22,231 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 01:21:22,232 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 01:21:22,348 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:21:27,349 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:21:27,879 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:21:32,350 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:21:32,880 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:21:37,231 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 01:21:37,232 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 01:21:37,398 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:21:42,403 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:21:42,883 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:21:43,918 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 01:21:43,920 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 01:21:43,920 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 01:21:43,921 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 01:21:44,033 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 01:21:45,034 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 01:21:47,888 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:21:47,922 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:21:49,502 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 01:21:52,231 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 01:21:52,232 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 01:21:53,395 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:21:57,890 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:21:58,395 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:22:02,891 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:22:03,396 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:22:07,231 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 01:22:07,232 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 01:22:09,311 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:22:12,897 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:22:14,060 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 01:22:14,062 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 01:22:14,062 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 01:22:14,064 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 01:22:15,060 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 01:22:15,065 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:22:16,060 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 01:22:17,898 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:22:19,506 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 01:22:20,508 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:22:22,231 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 01:22:22,232 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 01:22:26,400 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:22:27,338 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:22:31,400 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:22:32,339 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:22:36,401 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:22:37,232 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 01:22:37,232 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 01:22:42,353 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:22:42,360 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:22:43,982 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 01:22:43,983 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 01:22:43,984 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 01:22:43,985 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 01:22:44,084 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 01:22:46,085 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 01:22:47,398 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:22:47,986 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:22:49,509 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 01:22:52,232 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 01:22:52,232 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 01:22:53,362 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:22:57,400 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:22:58,363 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:23:02,401 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:23:03,363 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:23:07,232 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 01:23:07,232 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 01:23:09,307 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:23:12,402 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:23:14,308 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:23:17,412 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:23:19,312 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:23:19,512 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 01:23:19,794 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 01:23:19,795 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 01:23:19,795 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 01:23:19,797 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 01:23:20,117 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 01:23:22,119 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 01:23:22,232 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 01:23:22,233 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 01:23:25,304 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:23:27,411 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:23:30,305 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:23:32,642 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:23:35,305 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:23:37,232 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 01:23:37,233 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 01:23:40,313 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:23:42,644 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:23:45,314 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:23:47,648 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:23:49,519 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 01:23:50,520 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:23:51,379 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 01:23:51,380 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 01:23:51,380 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 01:23:51,381 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 01:23:52,149 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 01:23:52,232 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 01:23:52,233 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 01:23:54,150 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 01:23:56,336 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:23:57,825 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:24:01,336 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:24:02,826 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:24:06,337 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:24:07,232 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 01:24:07,233 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 01:24:11,368 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:24:12,870 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:24:16,379 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:24:17,882 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:24:19,526 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 01:24:21,529 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:24:22,232 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 01:24:22,233 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 01:24:22,240 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 01:24:22,361 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 01:24:22,362 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 01:24:22,363 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 01:24:23,176 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 01:24:24,177 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 01:24:27,365 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:24:27,890 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:24:32,365 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:24:32,891 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:24:37,233 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 01:24:37,233 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 01:24:38,320 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:24:42,893 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:24:43,322 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:24:47,899 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:24:48,325 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:24:49,528 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 01:24:52,233 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 01:24:52,233 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 01:24:52,410 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 01:24:52,412 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 01:24:52,413 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 01:24:52,414 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 01:24:53,208 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 01:24:53,415 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:24:54,211 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 01:24:57,901 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:24:58,416 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:25:02,902 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:25:03,417 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:25:07,233 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 01:25:07,233 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 01:25:09,363 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:25:12,903 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:25:14,363 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:25:17,904 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:25:19,364 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:25:19,546 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 01:25:22,239 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 01:25:22,243 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 01:25:24,364 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:25:25,310 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 01:25:25,311 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 01:25:25,311 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 01:25:25,313 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 01:25:26,239 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 01:25:27,913 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:25:28,240 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 01:25:30,314 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:25:32,914 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:25:35,314 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:25:37,235 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 01:25:37,235 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 01:25:40,344 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:25:42,996 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:25:45,345 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:25:47,996 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:25:47,997 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:25:49,538 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 01:25:50,548 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:25:52,237 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 01:25:52,238 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 01:25:56,294 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 01:25:56,296 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 01:25:56,296 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 01:25:56,297 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:25:56,302 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 01:25:57,268 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 01:25:58,003 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:25:58,269 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 01:26:01,304 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:26:03,004 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:26:06,304 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:26:07,237 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 01:26:07,238 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 01:26:11,349 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:26:13,005 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:26:16,350 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:26:18,006 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:26:18,007 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:26:19,542 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 01:26:21,544 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:26:22,238 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 01:26:22,238 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 01:26:27,303 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:26:28,023 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:26:30,073 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 01:26:30,075 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 01:26:30,076 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 01:26:30,081 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 01:26:30,296 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 01:26:32,297 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 01:26:33,016 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:26:33,082 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:26:37,238 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 01:26:37,238 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 01:26:38,350 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:26:43,018 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:26:43,351 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:26:48,019 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:26:48,351 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:26:49,548 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 01:26:52,238 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 01:26:52,239 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 01:26:53,357 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:26:58,027 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:26:58,298 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 01:26:58,300 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 01:26:58,300 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 01:26:58,301 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 01:26:58,318 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 01:26:59,302 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:27:00,320 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 01:27:03,028 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:27:04,303 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:27:07,238 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 01:27:07,238 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 01:27:09,334 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:27:13,030 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:27:14,335 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:27:18,031 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:27:19,335 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:27:19,550 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 01:27:22,238 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 01:27:22,239 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 01:27:24,347 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:27:28,042 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:27:29,348 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:27:30,150 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 01:27:30,152 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 01:27:30,153 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 01:27:30,156 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 01:27:30,351 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 01:27:32,354 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 01:27:33,039 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:27:35,158 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:27:37,238 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 01:27:37,239 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 01:27:40,376 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:27:43,183 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:27:43,264 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:27:45,376 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:27:48,265 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:27:49,554 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 01:27:50,556 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:27:52,238 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 01:27:52,239 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 01:27:56,408 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:27:57,273 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:28:00,922 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 01:28:00,924 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 01:28:00,924 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 01:28:00,926 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 01:28:01,385 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 01:28:01,927 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:28:02,286 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:28:02,286 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:28:02,387 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 01:28:06,927 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:28:07,238 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 01:28:07,239 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 01:28:12,288 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:28:12,288 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:28:12,346 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:28:17,289 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:28:17,290 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:28:17,346 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:28:19,558 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 01:28:22,238 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 01:28:22,239 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 01:28:23,344 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:28:27,299 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:28:28,351 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:28:30,047 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 01:28:30,048 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 01:28:30,048 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 01:28:30,050 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 01:28:30,410 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 01:28:32,412 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 01:28:32,444 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:28:34,051 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:28:37,239 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 01:28:37,240 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 01:28:39,309 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:28:42,464 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:28:44,310 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:28:47,464 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:28:49,311 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:28:49,562 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 01:28:52,239 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 01:28:52,239 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 01:28:54,331 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:28:57,471 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:28:59,315 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 01:28:59,317 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 01:28:59,317 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 01:28:59,319 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 01:28:59,437 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 01:29:00,320 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:29:00,438 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 01:29:02,722 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:29:02,722 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:29:05,322 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:29:07,239 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 01:29:07,239 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 01:29:11,304 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:29:12,723 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:29:12,724 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:29:16,305 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:29:17,724 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:29:17,725 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:29:19,567 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 01:29:21,569 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:29:22,239 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 01:29:22,239 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 01:29:27,367 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:29:27,735 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:29:29,123 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 01:29:29,125 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 01:29:29,125 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 01:29:29,126 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 01:29:29,466 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 01:29:30,467 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 01:29:32,733 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:29:33,127 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:29:37,239 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 01:29:37,240 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 01:29:38,322 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:29:42,734 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:29:43,323 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:29:47,735 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:29:48,323 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:29:49,570 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 01:29:52,239 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 01:29:52,240 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 01:29:53,345 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:29:57,746 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:29:58,350 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:30:00,195 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 01:30:00,197 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 01:30:00,197 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 01:30:00,198 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 01:30:00,493 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 01:30:02,495 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 01:30:02,744 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:30:04,199 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:30:07,239 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 01:30:07,240 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 01:30:09,350 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:30:12,745 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:30:14,350 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:30:17,746 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:30:19,351 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:30:19,574 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 01:30:22,239 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 01:30:22,240 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 01:30:24,383 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:30:27,758 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:30:29,388 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:30:31,640 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 01:30:31,641 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 01:30:31,641 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 01:30:31,643 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 01:30:32,521 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 01:30:32,785 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:30:34,523 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 01:30:34,644 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:30:37,240 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 01:30:37,240 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 01:30:40,331 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:30:42,786 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:30:45,332 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:30:47,787 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:30:49,578 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 01:30:50,580 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:30:52,240 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 01:30:52,240 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 01:30:56,339 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:30:57,799 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:30:59,061 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 01:30:59,062 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 01:30:59,063 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 01:30:59,065 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 01:30:59,549 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 01:31:00,550 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 01:31:02,066 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:31:02,794 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:31:07,066 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:31:07,240 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 01:31:07,240 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 01:31:12,445 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:31:12,796 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:31:17,445 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:31:17,797 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:31:19,582 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 01:31:22,240 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 01:31:22,240 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 01:31:23,330 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:31:27,807 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:31:28,337 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:31:29,188 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 01:31:29,189 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 01:31:29,190 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 01:31:29,191 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 01:31:29,584 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 01:31:30,585 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 01:31:32,806 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:31:32,806 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:31:34,193 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:31:37,240 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 01:31:37,241 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 01:31:39,329 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:31:42,807 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:31:42,808 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:31:44,329 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:31:47,808 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:31:47,809 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:31:49,330 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:31:49,586 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 01:31:52,240 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 01:31:52,241 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 01:31:54,403 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:31:57,814 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:31:59,409 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:32:01,545 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 01:32:01,547 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 01:32:01,547 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 01:32:01,548 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 01:32:01,613 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 01:32:02,614 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 01:32:02,944 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:32:02,944 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:32:04,549 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:32:07,240 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 01:32:07,241 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 01:32:10,327 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:32:12,945 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:32:15,329 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:32:17,946 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:32:17,947 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:32:19,591 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 01:32:20,593 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:32:22,240 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 01:32:22,241 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 01:32:26,386 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:32:27,960 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:32:27,961 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:32:31,387 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:32:31,888 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 01:32:31,889 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 01:32:31,889 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 01:32:31,890 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 01:32:32,638 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 01:32:33,314 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:32:33,314 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:32:34,640 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 01:32:36,891 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:32:37,240 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 01:32:37,241 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 01:32:42,315 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:32:42,316 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:32:42,338 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:32:47,316 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:32:47,317 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:32:47,339 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:32:49,594 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 01:32:52,241 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 01:32:52,241 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 01:32:53,322 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:32:57,318 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:32:58,329 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:33:02,323 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:33:02,432 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 01:33:02,434 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 01:33:02,434 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 01:33:02,435 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 01:33:02,663 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 01:33:03,436 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:33:04,665 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 01:33:07,241 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 01:33:07,241 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 01:33:09,353 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:33:12,324 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:33:14,354 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:33:17,325 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:33:19,355 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:33:19,598 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 01:33:22,241 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 01:33:22,242 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 01:33:25,353 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:33:27,326 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:33:30,361 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:33:32,332 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:33:32,336 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 01:33:32,338 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 01:33:32,338 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 01:33:32,339 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 01:33:32,690 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 01:33:34,692 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 01:33:36,340 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:33:37,241 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 01:33:37,241 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 01:33:42,338 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:33:42,340 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:33:47,339 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:33:47,340 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:33:49,602 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 01:33:52,241 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 01:33:52,241 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 01:33:53,322 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:33:57,340 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:33:58,323 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:34:02,342 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:34:03,333 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:34:07,142 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 01:34:07,144 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 01:34:07,144 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 01:34:07,146 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 01:34:07,530 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:34:07,531 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:34:07,536 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 01:34:07,536 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 01:34:07,720 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 01:34:08,689 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:34:08,721 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 01:34:13,532 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:34:13,690 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:34:18,532 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:34:18,691 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:34:19,605 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 01:34:22,289 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 01:34:22,289 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 01:34:24,367 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:34:27,533 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:34:29,367 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:34:32,539 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:34:34,371 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:34:36,620 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 01:34:36,621 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 01:34:36,621 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 01:34:36,623 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 01:34:36,745 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 01:34:37,289 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 01:34:37,290 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 01:34:38,746 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 01:34:39,387 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:34:42,541 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:34:44,388 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:34:47,541 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:34:49,389 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:34:49,610 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 01:34:52,289 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 01:34:52,290 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 01:34:54,422 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:34:57,543 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:34:59,423 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:35:02,552 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:35:04,429 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:35:07,271 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 01:35:07,273 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 01:35:07,273 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 01:35:07,275 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 01:35:07,682 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:35:07,683 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:35:07,687 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 01:35:07,688 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 01:35:07,778 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 01:35:08,778 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 01:35:09,781 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:35:13,686 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:35:13,687 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:35:14,782 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:35:18,687 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:35:19,613 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 01:35:20,615 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:35:22,418 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 01:35:22,418 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 01:35:26,578 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:35:27,688 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:35:27,689 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:35:31,579 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:35:32,689 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:35:32,690 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:35:36,584 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:35:37,425 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 01:35:37,426 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 01:35:41,301 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 01:35:41,303 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 01:35:41,304 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 01:35:41,306 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 01:35:41,806 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 01:35:42,306 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:35:42,424 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:35:42,425 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:35:42,807 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 01:35:47,307 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:35:47,426 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:35:49,617 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 01:35:52,422 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 01:35:52,423 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 01:35:52,535 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:35:57,496 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:35:57,497 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:35:57,536 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:36:02,497 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:36:02,497 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:36:02,536 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:36:07,422 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 01:36:07,423 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 01:36:08,513 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:36:12,505 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:36:13,524 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:36:14,584 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 01:36:14,586 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 01:36:14,586 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 01:36:14,587 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 01:36:14,831 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 01:36:16,833 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 01:36:17,502 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:36:18,588 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:36:19,621 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 01:36:22,422 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 01:36:22,423 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 01:36:24,557 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:36:27,504 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:36:29,558 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:36:32,505 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:36:34,559 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:36:37,422 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 01:36:37,423 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 01:36:40,508 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:36:42,515 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:36:45,496 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 01:36:45,497 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 01:36:45,497 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 01:36:45,499 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 01:36:45,859 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 01:36:46,499 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:36:46,860 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 01:36:47,642 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:36:49,625 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 01:36:51,628 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:36:52,423 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 01:36:52,423 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 01:36:57,491 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:36:57,643 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:37:02,492 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:37:02,644 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:37:07,423 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 01:37:07,423 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 01:37:07,559 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:37:12,563 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:37:12,648 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:37:17,417 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 01:37:17,418 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 01:37:17,418 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 01:37:17,419 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 01:37:17,822 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:37:17,888 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 01:37:18,420 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:37:18,890 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 01:37:19,629 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 01:37:22,423 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 01:37:22,423 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 01:37:23,510 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:37:27,824 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:37:28,510 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:37:32,824 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:37:33,511 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:37:37,423 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 01:37:37,423 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 01:37:38,566 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:37:42,833 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:37:43,570 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:37:47,832 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:37:48,371 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 01:37:48,373 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 01:37:48,373 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 01:37:48,375 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 01:37:48,917 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 01:37:48,917 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 01:37:49,375 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:37:49,633 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 01:37:52,423 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 01:37:52,423 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 01:37:54,567 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:37:57,833 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:37:59,568 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:38:02,835 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:38:04,568 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:38:07,423 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 01:38:07,424 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 01:38:10,537 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:38:12,845 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:38:15,547 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:38:17,584 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 01:38:17,585 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 01:38:17,585 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 01:38:17,587 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 01:38:17,943 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 01:38:18,017 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:38:18,944 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 01:38:19,637 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 01:38:20,639 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:38:22,423 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 01:38:22,424 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 01:38:26,570 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:38:28,018 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:38:31,571 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:38:33,019 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:38:36,571 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:38:37,423 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 01:38:37,424 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 01:38:42,512 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:38:43,024 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:38:47,514 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:38:47,899 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 01:38:47,901 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 01:38:47,901 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 01:38:47,902 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 01:38:47,975 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 01:38:48,054 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:38:48,300 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:38:48,975 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 01:38:49,641 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 01:38:52,423 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 01:38:52,424 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 01:38:52,592 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:38:57,593 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:38:58,301 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:39:02,594 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:39:03,376 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:39:03,376 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:39:07,424 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 01:39:07,424 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 01:39:08,491 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:39:13,386 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:39:13,501 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:39:15,119 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 01:39:15,120 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 01:39:15,120 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 01:39:15,122 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 01:39:15,996 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 01:39:16,997 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 01:39:18,531 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:39:18,531 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:39:19,123 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:39:19,645 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 01:39:22,424 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 01:39:22,424 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 01:39:24,569 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:39:27,532 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:39:27,533 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:39:29,570 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:39:32,533 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:39:32,533 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:39:34,571 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:39:37,424 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 01:39:37,424 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 01:39:39,578 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:39:42,543 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:39:44,576 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:39:45,536 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 01:39:45,538 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 01:39:45,538 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 01:39:45,540 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 01:39:46,030 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 01:39:47,031 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 01:39:47,684 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:39:49,649 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 01:39:49,655 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:39:52,424 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 01:39:52,424 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 01:39:55,507 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:39:57,685 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:40:00,508 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:40:02,686 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:40:05,508 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:40:07,424 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 01:40:07,425 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 01:40:10,576 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:40:12,691 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:40:15,589 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:40:17,046 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 01:40:17,048 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 01:40:17,048 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 01:40:17,049 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 01:40:17,058 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 01:40:17,691 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:40:19,060 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 01:40:19,654 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 01:40:20,656 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:40:22,424 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 01:40:22,425 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 01:40:26,561 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:40:27,693 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:40:31,562 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:40:32,785 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:40:36,562 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:40:37,424 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 01:40:37,425 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 01:40:42,539 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:40:42,794 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:40:46,459 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 01:40:46,460 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 01:40:46,460 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 01:40:46,461 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 01:40:47,084 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 01:40:47,085 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 01:40:47,861 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:40:48,462 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:40:49,657 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 01:40:52,424 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 01:40:52,425 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 01:40:53,591 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:40:57,863 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:40:58,592 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:41:02,906 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:41:03,593 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:41:07,425 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 01:41:07,426 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 01:41:09,494 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:41:12,910 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:41:14,496 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:41:17,013 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 01:41:17,015 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 01:41:17,015 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 01:41:17,017 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 01:41:17,113 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 01:41:17,915 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:41:19,115 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 01:41:19,661 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 01:41:19,667 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:41:22,425 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 01:41:22,425 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 01:41:25,540 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:41:27,917 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:41:30,541 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:41:32,918 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:41:35,541 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:41:37,425 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 01:41:37,425 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 01:41:40,546 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:41:42,930 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:41:45,555 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:41:46,907 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 01:41:46,909 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 01:41:46,909 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 01:41:46,911 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 01:41:47,137 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 01:41:47,138 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 01:41:48,303 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:41:49,665 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 01:41:50,667 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:41:52,425 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 01:41:52,425 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 01:41:56,502 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:41:58,304 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:42:01,502 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:42:03,305 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:42:06,503 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:42:07,425 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 01:42:07,425 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 01:42:11,523 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:42:13,313 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:42:16,528 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:42:16,783 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 01:42:16,785 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 01:42:16,786 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 01:42:16,787 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 01:42:17,173 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 01:42:17,173 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 01:42:18,314 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:42:19,669 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 01:42:21,671 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:42:22,425 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 01:42:22,425 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 01:42:27,567 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:42:28,315 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:42:32,567 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:42:33,316 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:42:37,425 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 01:42:37,426 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 01:42:37,582 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:42:42,589 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:42:43,322 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:42:45,757 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 01:42:45,758 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 01:42:45,758 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 01:42:45,760 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 01:42:46,204 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 01:42:47,206 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 01:42:47,761 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:42:48,323 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:42:49,674 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 01:42:52,425 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 01:42:52,426 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 01:42:53,509 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:42:58,459 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:42:58,510 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:42:58,552 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:43:03,511 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:43:03,552 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:43:07,425 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 01:43:07,426 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 01:43:08,526 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:43:12,564 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:43:13,543 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:43:15,094 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 01:43:15,096 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 01:43:15,096 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 01:43:15,098 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 01:43:15,227 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 01:43:17,229 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 01:43:17,559 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:43:19,099 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:43:19,677 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 01:43:22,425 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 01:43:22,426 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 01:43:24,521 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:43:27,561 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:43:29,521 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:43:32,562 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:43:34,522 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:43:37,426 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 01:43:37,426 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 01:43:39,573 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:43:42,572 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:43:44,581 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:43:45,733 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 01:43:45,734 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 01:43:45,734 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 01:43:45,736 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 01:43:46,266 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 01:43:47,267 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 01:43:47,568 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:43:49,681 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 01:43:49,687 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:43:52,426 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 01:43:52,426 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 01:43:55,548 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:43:57,570 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:44:00,549 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:44:02,571 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:44:05,549 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:44:07,426 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 01:44:07,427 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 01:44:11,521 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:44:12,580 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:44:16,529 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:44:17,581 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:44:17,697 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 01:44:17,698 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 01:44:17,698 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 01:44:17,700 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 01:44:18,291 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 01:44:19,685 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 01:44:20,293 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 01:44:21,687 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:44:22,426 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 01:44:22,427 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 01:44:27,554 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:44:27,587 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:44:32,555 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:44:32,588 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:44:37,426 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 01:44:37,427 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 01:44:37,873 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:44:42,599 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:44:42,879 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:44:46,115 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 01:44:46,117 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 01:44:46,117 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 01:44:46,119 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 01:44:46,317 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 01:44:47,595 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:44:48,120 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:44:48,319 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 01:44:49,688 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 01:44:52,426 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 01:44:52,427 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 01:44:53,575 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:44:57,596 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:44:58,575 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:45:02,597 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:45:03,576 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:45:07,426 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 01:45:07,427 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 01:45:08,577 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:45:12,605 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:45:13,585 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:45:16,497 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 01:45:16,499 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 01:45:16,499 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 01:45:16,501 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 01:45:17,343 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 01:45:17,647 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:45:18,343 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 01:45:19,502 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:45:19,691 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 01:45:22,426 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 01:45:22,427 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 01:45:24,570 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:45:27,648 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:45:29,571 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:45:32,650 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:45:34,571 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:45:37,427 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 01:45:37,427 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 01:45:39,592 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:45:42,653 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:45:44,598 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:45:47,586 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 01:45:47,587 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 01:45:47,588 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 01:45:47,589 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 01:45:47,733 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:45:48,370 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 01:45:49,697 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 01:45:49,703 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:45:50,371 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 01:45:52,427 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 01:45:52,427 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 01:45:55,578 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:45:57,983 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:45:57,983 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:46:00,579 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:46:02,984 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:46:02,984 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:46:05,579 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:46:07,427 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 01:46:07,427 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 01:46:10,605 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:46:12,985 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:46:15,606 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:46:17,994 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:46:19,702 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 01:46:20,703 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:46:21,705 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 01:46:21,707 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 01:46:21,708 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 01:46:21,709 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 01:46:22,399 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 01:46:22,427 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 01:46:22,428 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 01:46:24,401 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 01:46:26,590 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:46:28,105 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:46:31,590 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:46:33,106 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:46:33,106 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:46:36,591 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:46:37,427 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 01:46:37,427 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 01:46:42,574 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:46:43,156 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:46:43,156 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:46:47,580 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:46:48,163 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:46:49,707 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 01:46:51,023 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 01:46:51,025 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 01:46:51,025 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 01:46:51,026 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 01:46:51,427 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 01:46:52,427 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 01:46:52,428 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 01:46:52,434 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 01:46:52,604 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:46:57,605 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:46:58,427 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:46:58,427 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:47:02,606 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:47:03,427 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:47:03,428 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:47:07,427 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 01:47:07,428 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 01:47:08,519 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:47:12,429 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:47:12,429 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:47:13,519 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:47:17,430 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:47:17,430 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:47:18,525 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:47:19,709 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 01:47:22,431 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 01:47:22,431 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 01:47:23,468 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 01:47:23,470 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 01:47:23,470 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 01:47:23,472 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 01:47:24,462 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 01:47:24,473 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:47:26,464 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 01:47:27,440 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:47:29,474 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:47:32,440 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:47:34,474 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:47:37,431 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 01:47:37,432 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 01:47:39,503 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:47:42,442 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:47:44,504 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:47:47,442 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:47:49,517 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:47:49,710 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 01:47:52,402 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 01:47:52,403 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 01:47:52,404 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 01:47:52,405 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 01:47:52,493 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 01:47:52,550 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:47:52,805 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:47:52,809 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 01:47:52,809 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 01:47:54,495 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 01:47:54,919 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:47:58,806 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:47:58,806 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:47:59,919 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:48:03,807 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:48:04,920 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:48:07,805 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 01:48:07,805 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 01:48:09,939 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:48:12,808 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:48:14,940 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:48:17,809 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:48:17,809 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:48:19,715 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 01:48:20,724 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:48:22,806 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 01:48:22,806 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 01:48:25,868 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:48:26,541 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 01:48:26,543 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 01:48:26,543 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 01:48:26,545 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 01:48:27,525 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 01:48:27,940 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:48:27,941 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:48:28,526 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 01:48:31,546 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:48:32,941 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:48:32,942 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:48:36,547 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:48:37,806 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 01:48:37,806 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 01:48:41,972 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:48:43,060 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:48:43,060 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:48:46,972 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:48:48,061 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:48:48,061 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:48:49,719 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 01:48:52,721 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:48:52,806 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 01:48:52,807 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 01:48:57,892 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:48:58,064 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:48:58,673 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 01:48:58,675 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 01:48:58,675 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 01:48:58,679 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 01:48:59,553 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 01:49:00,554 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 01:49:03,072 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:49:03,073 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:49:03,681 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:49:07,806 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 01:49:07,807 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 01:49:08,902 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:49:13,074 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:49:13,903 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:49:18,074 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:49:18,904 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:49:19,723 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 01:49:22,806 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 01:49:22,807 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 01:49:23,971 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:49:28,083 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:49:28,977 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:49:30,071 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 01:49:30,072 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 01:49:30,072 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 01:49:30,075 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 01:49:30,580 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 01:49:32,581 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 01:49:33,483 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:49:33,483 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:49:34,076 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:49:37,806 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 01:49:37,807 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 01:49:39,928 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:49:43,484 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:49:43,484 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:49:44,929 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:49:48,485 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:49:48,486 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:49:49,727 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 01:49:50,729 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:49:52,807 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 01:49:52,807 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 01:49:55,969 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:49:58,490 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:50:00,984 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:50:02,620 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 01:50:02,621 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 01:50:02,622 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 01:50:02,627 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 01:50:03,494 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:50:03,494 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:50:03,611 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 01:50:04,611 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 01:50:06,628 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:50:07,807 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 01:50:07,808 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 01:50:11,880 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:50:13,496 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:50:13,496 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:50:16,881 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:50:18,497 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:50:18,497 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:50:19,731 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 01:50:22,733 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:50:22,807 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 01:50:22,807 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 01:50:27,966 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:50:28,504 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:50:30,809 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 01:50:30,810 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 01:50:30,811 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 01:50:30,812 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 01:50:31,640 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 01:50:32,641 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 01:50:33,502 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:50:33,813 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:50:37,807 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 01:50:37,808 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 01:50:38,924 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:50:43,503 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:50:43,925 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:50:48,504 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:50:48,925 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:50:49,734 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 01:50:52,807 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 01:50:52,808 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 01:50:54,922 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:50:58,515 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:50:59,931 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:51:01,494 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 01:51:01,495 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 01:51:01,495 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 01:51:01,497 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 01:51:01,669 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 01:51:02,670 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 01:51:03,897 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:51:03,897 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:51:05,498 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:51:07,807 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 01:51:07,808 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 01:51:10,942 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:51:12,898 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:51:12,899 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:51:15,943 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:51:17,900 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:51:17,900 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:51:19,739 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 01:51:21,742 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:51:22,807 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 01:51:22,808 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 01:51:26,923 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:51:27,910 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:51:30,546 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 01:51:30,548 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 01:51:30,548 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 01:51:30,549 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 01:51:30,696 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 01:51:32,550 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:51:32,698 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 01:51:32,951 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:51:32,953 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:51:37,551 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:51:37,807 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 01:51:37,808 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 01:51:42,920 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:51:42,954 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:51:47,921 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:51:47,955 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:51:47,955 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:51:49,742 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 01:51:52,808 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 01:51:52,808 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 01:51:53,895 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:51:57,959 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:51:58,901 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:52:00,985 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 01:52:00,986 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 01:52:00,987 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 01:52:00,988 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 01:52:01,727 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 01:52:02,728 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 01:52:02,964 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:52:03,989 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:52:07,808 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 01:52:07,809 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 01:52:09,975 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:52:12,965 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:52:14,976 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:52:17,966 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:52:19,746 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 01:52:20,749 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:52:22,808 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 01:52:22,808 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 01:52:25,889 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:52:27,978 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:52:30,898 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:52:32,976 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:52:33,475 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 01:52:33,476 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 01:52:33,477 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 01:52:33,478 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 01:52:33,761 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 01:52:34,762 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 01:52:36,479 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:52:37,808 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 01:52:37,809 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 01:52:41,926 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:52:42,978 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:52:46,927 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:52:47,979 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:52:47,979 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:52:49,750 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 01:52:52,753 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:52:52,808 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 01:52:52,808 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 01:52:57,973 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:52:57,989 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:53:02,975 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:53:02,986 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:53:03,442 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 01:53:03,444 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 01:53:03,444 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 01:53:03,446 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 01:53:03,791 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 01:53:04,792 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 01:53:07,808 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 01:53:07,809 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 01:53:08,906 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:53:12,988 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:53:13,907 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:53:18,043 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:53:18,044 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:53:18,907 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:53:19,754 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 01:53:22,808 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 01:53:22,809 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 01:53:23,918 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:53:28,045 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:53:28,045 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:53:28,918 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:53:33,054 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:53:33,931 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:53:35,586 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 01:53:35,587 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 01:53:35,587 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 01:53:35,589 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 01:53:35,821 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 01:53:36,822 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 01:53:37,808 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 01:53:37,809 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 01:53:38,937 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:53:43,052 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:53:43,053 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:53:43,938 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:53:48,053 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:53:48,054 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:53:48,938 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:53:49,759 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 01:53:52,808 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 01:53:52,809 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 01:53:54,870 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:53:58,055 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:53:59,870 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:54:03,068 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:54:03,068 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:54:04,878 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:54:06,934 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 01:54:06,936 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 01:54:06,936 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 01:54:06,938 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 01:54:07,809 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 01:54:07,810 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 01:54:07,852 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 01:54:08,853 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 01:54:09,921 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:54:13,081 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:54:14,922 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:54:18,082 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:54:18,082 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:54:19,759 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 01:54:20,762 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:54:22,809 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 01:54:22,809 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 01:54:25,903 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:54:28,130 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:54:30,904 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:54:33,132 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:54:35,910 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:54:37,811 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 01:54:37,811 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 01:54:38,555 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 01:54:38,556 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 01:54:38,557 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 01:54:38,561 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 01:54:38,880 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 01:54:40,882 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 01:54:41,562 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:54:43,138 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:54:46,563 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:54:48,138 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:54:49,763 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 01:54:51,766 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:54:52,811 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 01:54:52,811 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 01:54:56,938 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:54:58,140 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:55:01,938 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:55:03,141 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:55:06,945 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:55:07,817 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 01:55:07,818 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 01:55:10,125 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 01:55:10,127 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 01:55:10,127 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 01:55:10,130 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 01:55:10,905 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 01:55:12,131 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:55:12,907 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 01:55:13,523 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:55:13,523 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:55:17,132 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:55:18,524 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:55:18,524 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:55:19,767 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 01:55:22,770 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:55:22,935 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 01:55:22,936 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 01:55:28,094 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:55:28,526 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:55:28,526 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:55:33,095 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:55:33,526 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:55:33,526 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:55:37,842 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 01:55:37,842 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 01:55:38,954 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:55:40,428 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 01:55:40,430 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 01:55:40,431 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 01:55:40,435 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 01:55:40,934 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 01:55:42,935 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 01:55:43,322 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:55:43,322 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:55:44,436 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:55:48,323 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:55:48,323 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:55:49,437 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:55:49,771 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 01:55:52,842 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 01:55:52,843 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 01:55:55,002 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:55:58,526 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:55:58,527 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:56:00,003 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:56:03,527 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:56:03,527 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:56:05,003 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:56:07,846 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 01:56:07,849 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 01:56:10,974 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:56:11,200 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 01:56:11,202 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 01:56:11,203 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 01:56:11,206 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 01:56:11,960 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 01:56:12,961 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 01:56:13,262 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:56:13,262 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:56:16,207 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:56:18,263 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:56:19,776 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 01:56:21,779 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:56:22,843 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 01:56:22,843 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 01:56:26,925 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:56:28,264 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:56:28,264 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:56:31,925 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:56:33,265 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:56:33,265 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:56:36,928 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:56:37,843 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 01:56:37,846 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 01:56:42,005 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:56:42,318 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 01:56:42,319 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 01:56:42,320 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 01:56:42,325 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 01:56:42,986 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 01:56:43,281 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:56:43,282 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:56:44,988 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 01:56:47,326 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:56:48,282 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:56:49,778 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 01:56:52,780 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:56:52,843 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 01:56:52,843 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 01:56:57,995 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:56:58,284 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:56:58,284 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:57:02,996 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:57:03,285 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:57:07,843 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 01:57:07,844 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 01:57:08,006 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:57:13,017 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:57:13,293 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:57:15,342 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 01:57:15,343 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 01:57:15,344 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 01:57:15,347 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 01:57:16,017 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 01:57:17,018 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 01:57:18,288 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:57:18,402 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:57:19,782 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 01:57:22,843 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 01:57:22,844 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 01:57:24,002 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:57:28,289 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:57:29,003 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:57:33,290 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:57:34,003 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:57:37,843 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 01:57:37,844 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 01:57:39,960 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:57:43,306 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:57:44,957 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 01:57:44,959 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 01:57:44,960 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 01:57:44,960 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:57:44,961 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 01:57:45,048 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 01:57:47,049 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 01:57:48,366 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:57:48,366 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:57:49,786 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 01:57:50,788 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:57:52,843 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 01:57:52,844 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 01:57:55,923 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:57:58,367 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:58:00,924 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:58:03,369 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:58:03,369 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:58:05,924 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:58:07,843 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 01:58:07,844 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 01:58:11,019 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:58:13,376 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:58:14,983 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 01:58:14,984 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 01:58:14,985 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 01:58:14,986 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 01:58:15,069 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 01:58:16,987 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:58:17,071 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 01:58:18,388 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:58:18,388 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:58:19,790 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 01:58:22,793 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:58:22,844 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 01:58:22,844 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 01:58:27,991 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:58:28,389 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:58:32,992 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:58:33,390 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:58:37,844 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 01:58:37,844 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 01:58:38,918 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:58:43,398 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:58:43,923 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:58:46,692 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 01:58:46,694 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 01:58:46,694 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 01:58:46,699 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 01:58:47,096 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 01:58:48,399 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:58:49,097 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 01:58:49,700 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:58:49,794 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 01:58:52,844 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 01:58:52,844 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 01:58:54,948 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:58:58,400 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:58:59,949 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:59:03,401 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:59:04,950 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:59:07,844 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 01:59:07,844 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 01:59:09,951 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:59:13,411 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:59:14,961 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:59:17,385 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 01:59:17,387 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 01:59:17,387 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 01:59:17,392 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 01:59:18,126 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 01:59:18,793 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:59:18,793 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:59:19,127 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 01:59:19,799 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 01:59:20,801 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:59:22,844 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 01:59:22,845 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 01:59:26,002 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:59:28,794 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:59:28,794 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:59:31,002 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:59:33,796 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:59:36,004 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:59:37,844 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 01:59:37,844 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 01:59:41,994 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:59:43,803 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:59:47,002 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:59:48,807 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:59:49,529 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 01:59:49,530 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 01:59:49,530 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 01:59:49,532 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 01:59:49,802 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 01:59:50,157 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 01:59:51,158 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 01:59:52,804 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 01:59:52,844 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 01:59:52,846 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 01:59:57,931 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:59:57,932 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 01:59:58,012 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:00:02,932 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:00:02,933 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:00:03,012 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:00:07,845 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 02:00:07,845 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 02:00:09,010 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:00:12,934 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:00:12,934 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:00:14,010 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:00:17,935 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:00:19,019 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:00:19,809 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 02:00:22,844 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 02:00:22,845 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 02:00:22,894 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 02:00:22,992 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 02:00:22,993 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 02:00:22,995 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 02:00:23,189 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 02:00:24,996 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:00:25,190 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 02:00:28,294 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:00:29,997 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:00:33,294 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:00:34,998 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:00:37,845 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 02:00:37,845 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 02:00:40,018 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:00:43,296 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:00:43,296 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:00:45,019 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:00:48,299 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:00:49,817 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 02:00:50,820 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:00:52,014 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 02:00:52,016 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 02:00:52,016 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 02:00:52,018 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 02:00:52,210 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 02:00:52,848 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 02:00:52,849 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 02:00:53,211 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 02:00:56,006 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:00:58,428 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:00:58,428 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:01:01,007 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:01:03,429 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:01:03,429 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:01:06,007 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:01:07,848 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 02:01:07,848 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 02:01:11,975 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:01:13,430 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:01:16,977 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:01:18,439 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:01:19,817 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 02:01:21,147 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 02:01:21,149 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 02:01:21,149 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 02:01:21,151 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 02:01:21,243 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 02:01:22,152 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:01:22,848 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 02:01:22,848 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 02:01:23,245 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 02:01:27,945 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:01:28,547 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:01:28,547 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:01:32,946 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:01:33,548 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:01:33,549 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:01:37,848 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 02:01:37,849 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 02:01:38,004 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:01:43,004 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:01:43,550 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:01:43,550 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:01:48,017 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:01:48,560 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:01:49,817 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 02:01:49,818 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 02:01:49,819 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 02:01:49,819 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 02:01:49,820 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 02:01:50,274 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 02:01:51,274 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 02:01:52,848 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 02:01:52,849 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 02:01:53,983 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:01:58,559 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:01:58,983 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:02:03,559 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:02:03,984 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:02:07,848 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 02:02:07,849 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 02:02:09,017 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:02:13,560 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:02:14,021 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:02:18,577 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:02:18,577 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:02:19,026 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:02:19,817 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 02:02:19,876 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 02:02:19,877 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 02:02:19,877 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 02:02:19,878 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 02:02:20,297 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 02:02:21,298 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 02:02:22,848 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 02:02:22,849 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 02:02:24,940 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:02:28,571 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:02:29,941 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:02:33,572 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:02:34,942 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:02:37,848 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 02:02:37,849 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 02:02:39,956 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:02:43,573 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:02:44,957 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:02:48,577 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:02:49,819 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 02:02:50,820 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:02:51,169 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 02:02:51,170 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 02:02:51,171 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 02:02:51,174 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 02:02:51,325 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 02:02:52,848 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 02:02:52,849 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 02:02:53,332 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 02:02:55,979 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:02:58,582 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:03:00,980 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:03:03,582 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:03:05,980 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:03:07,849 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 02:03:07,849 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 02:03:11,927 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:03:13,706 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:03:13,706 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:03:16,927 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:03:18,707 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:03:18,707 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:03:19,824 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 02:03:22,835 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:03:22,856 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 02:03:22,860 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 02:03:25,753 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 02:03:25,755 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 02:03:25,756 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 02:03:25,761 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 02:03:26,361 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 02:03:27,362 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 02:03:28,713 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:03:28,713 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:03:28,762 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:03:33,714 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:03:33,714 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:03:33,763 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:03:37,852 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 02:03:37,853 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 02:03:39,020 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:03:43,715 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:03:43,716 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:03:44,021 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:03:48,716 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:03:48,716 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:03:49,021 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:03:49,832 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 02:03:52,856 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 02:03:52,856 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 02:03:54,930 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:03:55,120 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 02:03:55,122 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 02:03:55,122 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 02:03:55,124 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 02:03:55,387 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 02:03:57,388 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 02:03:58,725 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:04:00,127 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:04:03,725 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:04:05,127 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:04:07,856 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 02:04:07,857 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 02:04:10,988 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:04:13,727 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:04:15,988 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:04:18,728 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:04:19,833 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 02:04:21,838 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:04:22,863 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 02:04:22,864 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 02:04:25,784 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 02:04:25,786 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 02:04:25,786 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 02:04:25,787 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 02:04:26,411 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 02:04:27,412 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 02:04:27,788 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:04:28,734 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:04:32,789 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:04:33,734 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:04:37,789 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:04:37,860 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 02:04:37,860 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 02:04:42,940 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:04:43,736 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:04:47,940 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:04:48,736 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:04:49,838 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 02:04:52,869 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 02:04:52,873 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 02:04:53,020 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:04:57,508 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 02:04:57,510 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 02:04:57,511 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 02:04:57,515 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 02:04:57,917 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:04:57,917 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:04:58,444 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 02:04:58,516 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:04:59,445 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 02:05:02,918 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:05:03,517 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:05:07,864 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 02:05:07,865 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 02:05:08,968 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:05:12,919 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:05:13,968 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:05:17,920 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:05:17,920 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:05:18,969 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:05:19,841 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 02:05:22,871 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 02:05:22,872 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 02:05:23,976 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:05:27,109 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 02:05:27,111 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 02:05:27,111 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 02:05:27,113 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 02:05:27,471 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 02:05:27,929 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:05:29,114 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:05:29,473 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 02:05:32,930 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:05:34,115 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:05:37,868 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 02:05:37,869 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 02:05:39,941 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:05:42,931 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:05:44,942 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:05:47,932 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:05:49,845 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 02:05:50,852 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:05:52,874 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 02:05:52,875 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 02:05:55,193 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 02:05:55,195 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 02:05:55,195 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 02:05:55,196 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 02:05:55,501 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 02:05:55,502 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 02:05:56,197 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:05:57,939 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:06:01,197 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:06:02,939 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:06:06,198 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:06:07,870 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 02:06:07,870 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 02:06:11,945 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:06:12,941 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:06:16,945 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:06:17,941 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:06:19,849 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 02:06:22,860 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:06:22,877 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 02:06:22,885 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 02:06:26,095 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 02:06:26,096 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 02:06:26,096 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 02:06:26,098 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 02:06:26,526 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 02:06:27,527 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 02:06:27,948 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:06:28,099 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:06:32,948 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:06:33,100 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:06:37,873 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 02:06:37,874 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 02:06:38,938 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:06:42,976 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:06:43,939 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:06:47,976 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:06:48,939 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:06:49,853 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 02:06:52,877 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 02:06:52,880 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 02:06:54,025 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:06:56,437 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 02:06:56,439 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 02:06:56,439 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 02:06:56,441 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 02:06:56,555 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 02:06:57,560 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 02:06:57,983 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:06:59,442 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:07:02,984 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:07:04,443 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:07:07,874 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 02:07:07,875 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 02:07:09,993 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:07:12,985 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:07:14,993 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:07:17,986 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:07:19,858 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 02:07:20,351 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:07:22,574 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 02:07:25,355 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:07:29,002 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:07:29,481 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 02:07:29,482 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 02:07:30,579 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:07:35,579 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:07:37,590 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:07:37,806 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:07:39,122 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 02:07:39,123 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 02:07:41,276 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:07:46,277 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:07:46,476 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:07:46,650 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:07:49,861 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 02:07:51,863 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:07:55,094 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:07:55,498 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:07:56,007 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 02:07:56,008 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 02:07:57,175 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:08:02,175 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:08:03,523 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:08:03,696 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:08:07,176 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:08:12,176 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:08:12,330 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:08:16,685 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:08:16,686 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 02:08:16,686 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 02:08:17,796 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:08:17,926 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 02:08:19,865 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 02:08:19,927 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 02:08:21,686 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:08:21,687 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:08:22,867 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:08:26,687 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:08:26,688 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:08:27,467 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 02:08:27,468 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 02:08:28,565 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:08:32,689 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:08:32,689 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:08:33,566 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:08:37,690 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:08:37,690 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:08:38,567 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:08:42,467 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 02:08:42,468 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 02:08:43,584 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:08:47,692 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:08:47,692 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:08:48,584 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:08:49,868 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 02:08:52,693 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:08:52,693 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:08:53,870 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:08:57,467 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 02:08:57,468 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 02:08:59,547 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:09:02,695 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:09:02,695 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:09:04,547 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:09:07,695 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:09:07,696 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:09:09,548 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:09:14,549 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:09:19,549 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:09:19,872 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 02:09:24,874 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:09:29,875 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:09:31,893 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:09:31,897 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:09:31,897 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 02:09:31,897 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 02:09:35,057 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:09:36,898 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:09:36,898 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:09:40,057 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:09:41,899 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:09:41,900 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:09:45,058 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:09:46,897 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 02:09:46,898 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 02:09:49,875 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 02:09:50,878 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:09:51,901 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:09:51,901 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:09:55,878 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:09:56,902 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:09:56,902 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:10:00,879 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:10:01,897 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 02:10:01,898 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 02:10:06,067 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:10:06,903 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:10:06,904 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:10:11,068 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:10:11,905 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:10:11,905 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:10:16,068 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:10:16,897 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 02:10:16,898 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 02:10:19,880 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 02:10:21,883 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:10:21,907 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:10:21,907 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:10:26,883 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:10:26,908 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:10:26,908 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:10:31,884 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:10:31,897 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 02:10:31,898 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 02:10:36,910 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:10:36,910 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:10:36,993 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:10:41,911 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:10:41,911 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:10:41,994 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:10:46,898 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 02:10:46,898 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 02:10:47,970 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:10:49,884 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 02:10:51,912 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:10:51,913 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:10:53,886 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:10:56,913 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:10:56,914 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:10:58,887 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:11:01,898 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 02:11:01,899 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 02:11:04,034 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:11:06,916 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:11:06,916 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:11:09,034 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:11:11,917 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:11:11,917 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:11:14,035 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:11:16,898 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 02:11:16,898 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 02:11:19,887 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 02:11:19,894 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:11:21,918 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:11:21,919 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:11:24,890 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:11:26,919 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:11:26,920 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:11:29,891 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:11:34,892 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:11:39,892 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:11:44,893 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:11:49,892 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 02:11:49,898 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:11:54,894 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:11:59,895 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:12:04,896 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:12:08,534 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:12:08,538 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:12:08,539 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 02:12:08,539 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 02:12:10,676 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:12:12,170 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 02:12:13,778 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:12:13,778 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:12:16,000 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:12:16,173 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 02:12:18,779 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:12:18,779 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:12:19,895 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 02:12:21,898 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:12:23,534 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 02:12:23,535 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 02:12:27,720 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:12:28,781 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:12:28,781 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:12:32,720 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:12:33,782 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:12:33,782 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:12:37,724 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:12:38,542 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 02:12:38,546 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 02:12:42,741 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 02:12:42,742 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 02:12:42,743 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 02:12:42,743 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:12:42,744 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 02:12:43,199 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 02:12:44,143 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:12:44,143 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:12:44,200 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 02:12:47,746 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:12:49,144 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:12:49,144 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:12:49,900 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 02:12:52,903 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:12:53,538 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 02:12:53,539 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 02:12:58,703 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:12:59,146 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:12:59,146 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:13:03,704 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:13:04,147 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:13:08,545 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 02:13:08,546 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 02:13:09,682 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:13:13,335 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 02:13:13,336 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 02:13:13,336 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 02:13:13,338 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 02:13:14,153 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:13:14,223 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 02:13:14,224 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 02:13:15,338 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:13:19,154 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:13:19,904 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 02:13:20,906 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:13:23,543 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 02:13:23,544 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 02:13:26,678 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:13:29,156 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:13:31,678 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:13:34,157 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:13:36,679 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:13:38,551 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 02:13:38,552 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 02:13:41,721 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:13:43,119 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 02:13:43,121 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 02:13:43,122 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 02:13:43,123 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 02:13:43,250 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 02:13:44,251 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 02:13:44,266 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:13:44,266 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:13:47,125 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:13:49,267 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:13:49,267 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:13:49,908 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 02:13:52,910 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:13:53,547 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 02:13:53,547 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 02:13:58,648 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:13:59,269 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:13:59,269 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:14:03,649 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:14:04,270 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:14:04,270 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:14:08,547 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 02:14:08,548 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 02:14:09,629 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:14:14,280 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:14:14,639 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:14:16,989 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 02:14:16,991 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 02:14:16,991 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 02:14:16,992 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 02:14:17,283 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 02:14:18,284 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 02:14:19,403 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:14:19,912 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 02:14:19,918 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:14:23,547 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 02:14:23,548 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 02:14:25,631 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:14:29,404 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:14:30,631 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:14:34,405 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:14:35,632 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:14:38,548 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 02:14:38,548 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 02:14:40,670 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:14:44,415 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:14:45,681 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:14:47,142 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 02:14:47,143 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 02:14:47,144 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 02:14:47,145 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 02:14:47,311 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 02:14:48,312 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 02:14:49,551 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:14:49,916 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 02:14:50,918 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:14:53,548 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 02:14:53,548 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 02:14:56,650 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:14:58,552 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:15:01,651 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:15:03,553 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:15:06,651 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:15:08,548 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 02:15:08,549 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 02:15:12,645 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:15:13,569 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:15:14,679 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 02:15:14,680 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 02:15:14,681 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 02:15:14,682 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 02:15:15,335 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 02:15:16,336 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 02:15:17,683 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:15:18,565 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:15:18,565 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:15:19,919 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 02:15:22,922 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:15:23,548 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 02:15:23,548 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 02:15:28,567 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:15:28,567 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:15:28,700 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:15:33,568 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:15:33,568 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:15:33,701 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:15:38,548 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 02:15:38,549 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 02:15:38,716 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:15:43,573 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:15:43,723 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:15:46,881 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 02:15:46,883 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 02:15:46,883 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 02:15:46,885 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 02:15:47,368 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 02:15:48,369 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 02:15:48,574 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:15:48,886 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:15:49,924 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 02:15:53,548 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 02:15:53,549 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 02:15:54,687 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:15:58,576 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:15:59,688 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:16:03,753 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:16:04,688 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:16:08,548 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 02:16:08,549 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 02:16:09,714 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:16:13,761 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:16:14,733 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:16:17,201 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 02:16:17,202 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 02:16:17,202 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 02:16:17,204 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 02:16:17,392 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 02:16:18,393 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 02:16:18,761 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:16:19,927 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 02:16:19,934 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:16:23,548 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 02:16:23,549 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 02:16:25,669 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:16:28,762 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:16:30,669 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:16:33,763 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:16:35,670 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:16:38,548 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 02:16:38,549 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 02:16:40,715 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:16:43,766 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:16:45,724 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:16:48,697 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 02:16:48,699 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 02:16:48,699 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 02:16:48,700 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 02:16:48,842 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:16:49,121 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:16:49,424 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 02:16:49,931 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 02:16:50,425 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 02:16:50,934 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:16:53,549 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 02:16:53,549 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 02:16:56,653 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:16:59,122 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:16:59,122 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:17:01,653 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:17:04,123 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:17:04,124 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:17:06,654 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:17:08,549 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 02:17:08,549 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 02:17:12,633 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:17:14,133 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:17:17,642 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:17:18,402 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 02:17:18,403 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 02:17:18,403 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 02:17:18,405 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 02:17:18,445 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 02:17:19,133 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:17:19,133 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:17:19,935 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 02:17:20,446 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 02:17:22,938 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:17:23,549 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 02:17:23,549 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 02:17:28,641 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:17:29,134 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:17:29,135 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:17:33,642 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:17:34,136 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:17:34,136 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:17:38,549 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 02:17:38,550 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 02:17:39,615 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:17:44,141 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:17:44,622 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:17:48,462 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 02:17:48,463 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 02:17:48,463 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 02:17:48,464 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 02:17:48,471 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 02:17:49,146 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:17:49,938 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 02:17:49,944 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:17:50,473 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 02:17:53,549 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 02:17:53,550 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 02:17:55,708 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:17:59,147 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:18:00,708 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:18:04,148 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:18:05,709 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:18:08,549 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 02:18:08,550 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 02:18:11,625 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:18:14,153 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:18:16,631 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:18:19,096 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 02:18:19,097 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 02:18:19,098 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 02:18:19,099 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 02:18:19,241 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:18:19,497 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 02:18:19,943 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 02:18:21,499 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 02:18:21,945 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:18:23,549 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 02:18:23,550 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 02:18:27,623 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:18:29,499 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:18:29,500 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:18:32,623 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:18:34,578 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:18:34,579 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:18:37,624 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:18:38,549 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 02:18:38,550 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 02:18:43,581 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:18:43,626 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:18:48,586 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:18:48,628 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:18:48,920 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 02:18:48,922 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 02:18:48,922 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 02:18:48,923 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 02:18:49,525 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 02:18:49,946 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 02:18:51,527 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 02:18:53,550 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 02:18:53,550 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 02:18:53,705 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:18:58,588 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:18:58,705 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:19:03,590 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:19:03,706 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:19:08,550 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 02:19:08,550 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 02:19:09,667 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:19:13,603 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:19:14,673 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:19:18,597 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:19:19,020 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 02:19:19,020 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 02:19:19,021 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 02:19:19,023 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 02:19:19,557 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 02:19:19,950 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 02:19:19,956 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:19:21,559 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 02:19:23,550 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 02:19:23,550 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 02:19:25,675 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:19:28,599 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:19:30,676 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:19:33,600 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:19:35,676 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:19:38,550 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 02:19:38,550 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 02:19:40,702 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:19:43,602 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:19:45,709 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:19:48,616 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:19:49,808 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 02:19:49,809 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 02:19:49,810 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 02:19:49,813 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 02:19:49,954 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 02:19:50,590 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 02:19:50,957 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:19:51,591 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 02:19:53,550 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 02:19:53,551 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 02:19:56,631 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:19:58,611 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:20:01,631 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:20:03,612 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:20:06,632 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:20:08,550 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 02:20:08,551 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 02:20:11,661 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:20:13,614 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:20:16,667 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:20:18,629 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:20:19,966 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 02:20:20,900 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 02:20:20,901 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 02:20:20,902 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 02:20:20,904 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 02:20:21,625 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 02:20:21,904 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:20:23,550 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 02:20:23,551 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 02:20:23,626 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 02:20:27,656 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:20:28,623 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:20:32,656 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:20:33,624 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:20:37,657 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:20:38,550 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 02:20:38,551 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 02:20:42,681 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:20:43,626 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:20:47,683 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:20:48,631 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:20:49,966 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 02:20:51,508 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 02:20:51,509 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 02:20:51,510 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 02:20:51,512 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 02:20:51,653 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 02:20:53,513 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:20:53,550 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 02:20:53,551 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 02:20:53,655 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 02:20:58,620 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:20:58,657 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:20:58,657 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:21:03,621 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:21:03,658 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:21:03,658 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:21:08,550 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 02:21:08,551 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 02:21:08,723 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:21:13,663 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:21:13,731 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:21:18,577 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 02:21:18,578 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 02:21:18,579 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 02:21:18,580 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 02:21:18,676 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 02:21:18,723 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:21:18,983 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:21:19,581 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:21:19,677 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 02:21:19,971 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 02:21:23,551 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 02:21:23,551 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 02:21:24,703 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:21:28,984 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:21:29,704 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:21:33,985 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:21:33,985 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:21:34,705 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:21:38,551 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 02:21:38,551 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 02:21:40,689 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:21:43,986 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:21:43,986 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:21:45,690 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:21:48,999 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:21:49,977 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 02:21:50,978 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:21:52,464 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 02:21:52,466 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 02:21:52,466 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 02:21:52,467 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 02:21:52,705 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 02:21:53,551 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 02:21:53,552 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 02:21:53,706 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 02:21:56,674 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:21:58,996 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:22:01,675 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:22:03,999 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:22:04,092 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:22:06,675 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:22:08,551 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 02:22:08,551 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 02:22:12,632 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:22:14,094 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:22:17,638 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:22:19,102 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:22:19,602 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 02:22:19,603 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 02:22:19,604 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 02:22:19,605 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 02:22:19,730 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 02:22:19,981 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 02:22:21,732 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 02:22:22,983 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:22:23,551 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 02:22:23,552 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 02:22:28,730 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:22:29,101 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:22:33,730 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:22:34,102 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:22:38,551 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 02:22:38,552 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 02:22:39,673 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:22:44,138 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:22:44,138 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:22:44,673 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:22:49,147 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:22:49,683 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:22:49,988 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 02:22:52,612 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 02:22:52,613 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 02:22:52,613 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 02:22:52,614 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 02:22:52,752 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 02:22:53,551 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 02:22:53,552 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 02:22:53,753 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 02:22:54,688 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:22:59,150 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:22:59,151 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:22:59,689 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:23:04,151 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:23:04,152 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:23:04,689 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:23:08,552 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 02:23:08,553 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 02:23:10,663 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:23:14,153 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:23:14,153 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:23:15,664 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:23:19,163 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:23:19,998 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 02:23:21,000 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:23:23,094 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 02:23:23,095 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 02:23:23,095 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 02:23:23,096 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 02:23:23,551 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 02:23:23,552 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 02:23:23,782 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 02:23:25,783 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 02:23:26,663 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:23:29,484 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:23:29,484 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:23:31,663 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:23:34,484 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:23:34,485 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:23:36,664 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:23:38,552 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 02:23:38,553 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 02:23:41,674 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:23:44,486 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:23:44,487 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:23:46,674 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:23:49,494 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:23:50,000 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 02:23:52,001 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:23:53,062 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 02:23:53,063 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 02:23:53,064 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 02:23:53,065 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 02:23:53,552 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 02:23:53,553 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 02:23:53,813 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 02:23:55,815 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 02:23:57,684 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:23:59,496 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:24:02,685 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:24:04,496 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:24:07,685 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:24:08,552 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 02:24:08,552 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 02:24:12,698 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:24:14,498 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:24:17,699 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:24:19,499 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:24:20,000 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 02:24:23,010 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:24:23,560 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 02:24:23,561 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 02:24:27,007 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 02:24:27,008 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 02:24:27,008 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 02:24:27,011 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 02:24:27,842 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 02:24:28,012 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:24:29,505 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:24:29,845 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 02:24:33,013 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:24:34,506 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:24:38,013 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:24:38,554 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 02:24:38,555 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 02:24:43,693 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:24:44,508 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:24:48,694 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:24:49,509 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:24:50,004 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 02:24:53,561 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 02:24:53,566 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 02:24:54,655 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:24:57,758 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 02:24:57,760 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 02:24:57,760 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 02:24:57,766 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 02:24:57,871 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 02:24:59,520 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:24:59,520 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:24:59,766 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:24:59,873 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 02:25:04,521 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:25:04,767 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:25:08,556 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 02:25:08,557 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 02:25:10,646 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:25:14,522 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:25:15,647 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:25:19,523 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:25:19,523 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:25:20,008 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 02:25:21,010 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:25:23,556 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 02:25:23,557 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 02:25:26,725 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:25:29,529 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:25:31,729 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:25:33,948 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 02:25:33,950 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 02:25:33,950 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 02:25:33,952 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 02:25:34,529 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:25:34,905 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 02:25:35,906 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 02:25:36,953 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:25:38,556 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 02:25:38,557 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 02:25:42,697 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:25:44,530 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:25:47,698 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:25:49,531 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:25:50,009 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 02:25:53,011 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:25:53,556 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 02:25:53,557 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 02:25:58,741 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:25:59,538 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:26:03,748 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:26:03,935 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 02:26:03,936 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 02:26:03,937 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 02:26:03,938 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 02:26:04,539 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:26:04,938 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 02:26:05,939 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 02:26:08,557 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 02:26:08,557 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 02:26:09,713 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:26:14,540 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:26:14,713 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:26:19,541 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:26:19,714 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:26:20,014 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 02:26:23,557 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 02:26:23,557 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 02:26:25,670 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:26:29,545 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:26:30,674 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:26:34,294 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 02:26:34,295 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 02:26:34,296 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 02:26:34,298 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 02:26:34,706 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:26:34,963 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 02:26:35,964 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 02:26:36,299 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:26:38,557 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 02:26:38,557 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 02:26:41,622 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:26:43,707 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:26:46,623 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:26:48,708 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:26:48,708 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:26:50,017 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 02:26:52,019 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:26:53,557 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 02:26:53,557 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 02:26:57,687 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:26:58,711 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:27:02,700 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:27:03,481 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 02:27:03,482 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 02:27:03,483 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 02:27:03,485 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 02:27:03,878 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:27:03,879 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:27:03,988 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 02:27:05,989 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 02:27:08,486 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:27:08,557 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 02:27:08,557 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 02:27:13,662 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:27:13,880 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:27:13,881 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:27:18,662 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:27:18,881 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:27:18,882 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:27:20,021 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 02:27:23,557 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 02:27:23,558 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 02:27:23,664 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:27:28,665 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:27:28,883 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:27:28,883 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:27:33,671 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:27:33,892 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:27:36,513 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 02:27:36,514 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 02:27:36,515 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 02:27:36,517 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 02:27:37,015 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 02:27:38,016 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 02:27:38,557 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 02:27:38,558 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 02:27:38,675 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:27:43,675 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:27:43,917 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:27:48,676 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:27:48,918 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:27:48,918 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:27:50,025 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 02:27:53,557 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 02:27:53,558 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 02:27:53,691 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:27:58,692 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:27:58,919 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:27:58,920 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:28:03,702 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:28:03,931 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:28:05,360 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 02:28:05,361 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 02:28:05,361 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 02:28:05,362 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 02:28:06,043 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 02:28:08,045 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 02:28:08,558 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 02:28:08,558 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 02:28:08,706 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:28:13,707 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:28:13,927 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:28:18,707 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:28:18,927 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:28:20,029 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 02:28:23,558 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 02:28:23,558 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 02:28:23,729 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:28:28,729 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:28:28,929 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:28:33,734 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:28:33,936 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:28:37,821 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 02:28:37,822 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 02:28:37,823 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 02:28:37,826 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 02:28:38,072 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 02:28:38,558 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 02:28:38,559 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 02:28:39,678 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:28:40,074 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 02:28:43,967 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:28:43,968 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:28:44,679 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:28:48,968 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:28:48,968 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:28:49,679 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:28:50,033 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 02:28:53,558 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 02:28:53,559 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 02:28:55,670 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:28:58,970 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:28:58,970 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:29:00,670 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:29:03,975 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:29:05,678 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:29:07,036 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 02:29:07,038 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 02:29:07,039 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 02:29:07,041 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 02:29:07,095 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 02:29:08,096 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 02:29:08,558 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 02:29:08,559 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 02:29:10,744 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:29:13,976 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:29:15,744 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:29:18,977 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:29:20,038 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 02:29:21,040 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:29:23,558 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 02:29:23,559 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 02:29:26,718 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:29:29,075 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:29:29,075 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:29:31,719 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:29:34,076 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:29:34,076 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:29:36,730 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:29:38,558 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 02:29:38,558 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 02:29:40,702 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 02:29:40,704 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 02:29:40,705 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 02:29:40,707 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 02:29:41,126 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 02:29:42,127 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 02:29:42,708 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:29:43,676 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:29:47,708 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:29:48,677 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:29:48,677 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:29:50,040 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 02:29:53,042 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:29:53,558 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 02:29:53,559 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 02:29:58,679 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:29:58,679 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:29:58,718 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:30:03,680 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:30:03,680 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:30:03,719 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:30:08,566 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 02:30:08,571 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 02:30:08,741 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:30:10,034 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 02:30:10,035 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 02:30:10,035 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 02:30:10,037 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 02:30:10,153 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 02:30:12,154 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 02:30:13,685 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:30:14,038 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:30:18,686 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:30:19,039 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:30:20,044 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 02:30:23,562 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 02:30:23,563 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 02:30:24,720 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:30:28,747 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:30:29,721 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:30:33,748 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:30:34,721 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:30:38,578 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 02:30:38,580 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 02:30:39,758 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:30:43,757 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:30:43,945 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 02:30:43,947 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 02:30:43,947 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 02:30:43,949 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 02:30:44,190 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 02:30:44,950 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:30:46,191 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 02:30:48,758 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:30:49,951 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:30:50,048 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 02:30:53,569 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 02:30:53,570 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 02:30:55,714 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:30:58,760 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:31:00,714 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:31:03,761 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:31:05,715 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:31:08,574 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 02:31:08,575 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 02:31:10,717 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:31:13,083 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 02:31:13,085 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 02:31:13,085 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 02:31:13,086 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 02:31:13,215 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 02:31:13,767 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:31:14,217 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 02:31:16,087 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:31:18,767 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:31:20,052 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 02:31:22,054 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:31:23,572 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 02:31:23,573 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 02:31:27,722 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:31:28,769 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:31:32,723 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:31:33,770 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:31:37,723 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:31:38,572 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 02:31:38,573 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 02:31:43,663 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:31:43,782 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:31:45,133 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 02:31:45,135 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 02:31:45,135 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 02:31:45,136 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 02:31:45,243 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 02:31:46,244 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 02:31:48,777 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:31:49,137 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:31:50,056 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 02:31:53,572 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 02:31:53,573 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 02:31:54,728 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:31:58,779 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:31:58,779 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:31:59,729 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:32:03,780 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:32:03,780 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:32:04,730 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:32:08,572 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 02:32:08,573 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 02:32:10,718 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:32:13,784 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:32:15,725 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:32:16,343 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 02:32:16,345 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 02:32:16,345 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 02:32:16,347 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 02:32:17,267 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 02:32:18,268 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 02:32:18,788 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:32:20,059 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 02:32:21,062 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:32:23,572 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 02:32:23,573 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 02:32:26,706 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:32:28,840 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:32:31,707 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:32:33,841 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:32:33,841 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:32:36,708 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:32:38,573 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 02:32:38,573 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 02:32:42,693 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:32:43,846 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:32:43,846 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:32:45,382 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 02:32:45,384 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 02:32:45,385 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 02:32:45,386 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 02:32:46,296 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 02:32:46,297 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 02:32:48,387 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:32:48,847 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:32:48,848 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:32:50,063 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 02:32:53,573 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 02:32:53,573 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 02:32:53,727 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:32:58,728 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:32:58,849 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:32:58,849 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:33:03,728 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:33:03,850 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:33:03,850 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:33:08,573 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 02:33:08,573 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 02:33:09,713 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:33:13,859 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:33:14,717 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:33:15,269 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 02:33:15,271 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 02:33:15,271 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 02:33:15,272 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 02:33:15,322 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 02:33:16,323 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 02:33:18,860 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:33:18,860 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:33:20,067 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 02:33:20,074 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:33:23,573 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 02:33:23,573 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 02:33:25,676 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:33:28,861 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:33:30,677 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:33:33,862 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:33:33,863 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:33:35,677 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:33:38,573 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 02:33:38,574 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 02:33:40,708 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:33:43,871 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:33:45,371 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 02:33:45,373 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 02:33:45,373 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 02:33:45,374 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 02:33:46,347 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 02:33:46,347 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 02:33:46,375 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:33:48,869 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:33:50,072 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 02:33:52,075 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:33:53,574 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 02:33:53,575 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 02:33:57,662 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:33:58,871 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:34:02,663 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:34:03,871 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:34:07,664 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:34:08,573 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 02:34:08,574 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 02:34:12,743 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:34:13,881 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:34:15,996 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 02:34:15,997 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 02:34:15,998 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 02:34:15,999 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 02:34:16,375 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 02:34:18,000 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:34:18,377 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 02:34:18,880 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:34:20,076 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 02:34:23,079 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:34:23,573 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 02:34:23,574 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 02:34:28,757 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:34:28,881 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:34:33,758 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:34:33,918 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:34:33,918 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:34:38,573 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 02:34:38,574 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 02:34:39,682 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:34:43,930 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:34:44,683 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:34:48,929 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:34:48,929 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:34:49,613 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 02:34:49,614 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 02:34:49,615 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 02:34:49,617 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 02:34:50,081 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 02:34:50,088 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:34:50,404 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 02:34:50,404 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 02:34:53,574 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 02:34:53,574 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 02:34:55,692 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:34:59,020 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:34:59,020 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:35:00,692 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:35:04,021 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:35:04,021 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:35:05,693 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:35:08,574 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 02:35:08,574 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 02:35:10,710 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:35:14,024 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:35:15,714 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:35:18,656 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 02:35:18,658 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 02:35:18,658 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 02:35:18,660 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 02:35:19,059 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:35:19,059 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:35:19,435 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 02:35:20,084 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 02:35:20,436 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 02:35:21,087 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:35:23,574 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 02:35:23,575 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 02:35:26,680 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:35:29,060 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:35:31,680 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:35:34,061 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:35:34,062 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:35:36,681 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:35:38,574 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 02:35:38,575 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 02:35:41,689 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:35:44,065 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:35:46,698 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:35:48,760 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 02:35:48,762 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 02:35:48,762 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 02:35:48,764 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 02:35:49,160 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:35:49,160 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:35:49,464 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 02:35:50,088 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 02:35:50,465 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 02:35:52,091 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:35:53,574 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 02:35:53,574 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 02:35:57,650 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:35:59,162 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:35:59,162 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:36:02,650 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:36:04,262 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:36:04,262 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:36:07,651 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:36:08,574 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 02:36:08,575 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 02:36:12,741 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:36:14,269 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:36:17,726 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 02:36:17,727 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 02:36:17,727 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 02:36:17,728 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 02:36:18,490 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 02:36:18,490 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 02:36:18,729 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:36:19,269 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:36:20,092 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 02:36:23,574 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 02:36:23,575 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 02:36:24,687 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:36:29,300 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:36:29,688 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:36:34,301 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:36:34,688 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:36:38,574 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 02:36:38,575 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 02:36:40,665 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:36:44,309 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:36:45,677 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:36:46,914 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 02:36:46,915 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 02:36:46,916 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 02:36:46,917 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 02:36:47,515 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 02:36:48,516 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 02:36:49,319 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:36:50,096 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 02:36:51,099 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:36:53,574 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 02:36:53,575 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 02:36:56,647 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:36:59,389 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:37:01,647 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:37:04,390 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:37:06,648 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:37:08,575 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 02:37:08,575 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 02:37:11,806 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:37:14,411 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:37:16,812 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:37:17,144 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 02:37:17,146 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 02:37:17,146 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 02:37:17,147 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 02:37:17,540 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 02:37:18,541 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 02:37:19,568 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:37:19,569 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:37:20,101 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 02:37:22,104 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:37:23,575 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 02:37:23,575 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 02:37:27,713 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:37:29,570 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:37:29,570 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:37:32,713 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:37:34,571 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:37:34,571 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:37:37,714 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:37:38,575 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 02:37:38,575 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 02:37:43,578 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:37:43,706 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:37:48,582 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:37:48,583 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:37:48,662 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 02:37:48,664 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 02:37:48,665 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 02:37:48,666 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 02:37:49,566 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 02:37:49,667 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:37:50,104 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 02:37:50,567 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 02:37:53,575 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 02:37:53,576 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 02:37:54,744 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:37:58,585 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:37:58,585 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:37:59,744 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:38:03,586 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:38:03,586 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:38:04,745 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:38:08,575 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 02:38:08,576 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 02:38:10,674 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:38:13,587 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:38:13,587 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:38:15,674 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:38:18,596 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:38:20,111 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 02:38:21,113 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:38:22,120 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 02:38:22,122 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 02:38:22,123 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 02:38:22,124 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 02:38:22,601 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 02:38:23,575 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 02:38:23,576 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 02:38:24,602 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 02:38:26,744 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:38:28,597 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:38:28,598 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:38:31,744 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:38:33,598 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:38:33,599 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:38:36,745 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:38:38,575 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 02:38:38,576 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 02:38:42,711 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:38:43,737 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:38:47,711 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:38:48,741 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:38:50,121 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 02:38:53,123 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:38:53,580 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 02:38:53,581 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 02:38:54,167 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 02:38:54,168 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 02:38:54,168 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 02:38:54,169 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 02:38:54,630 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 02:38:56,632 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 02:38:58,170 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:38:58,744 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:38:58,744 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:39:03,171 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:39:03,744 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:39:03,745 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:39:08,172 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:39:08,578 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 02:39:08,578 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 02:39:13,706 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:39:13,746 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:39:13,746 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:39:18,713 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:39:18,760 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:39:18,760 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:39:20,123 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 02:39:21,555 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 02:39:21,557 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 02:39:21,557 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 02:39:21,558 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 02:39:21,657 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 02:39:22,658 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 02:39:23,578 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 02:39:23,579 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 02:39:24,650 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:39:28,970 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:39:29,651 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:39:33,970 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:39:34,651 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:39:38,578 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 02:39:38,579 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 02:39:39,699 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:39:43,972 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:39:44,700 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:39:48,981 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:39:49,707 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:39:50,124 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 02:39:51,695 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 02:39:51,696 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 02:39:51,696 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 02:39:51,699 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 02:39:52,685 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 02:39:52,685 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 02:39:53,578 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 02:39:53,579 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 02:39:54,750 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:39:59,097 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:39:59,098 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:39:59,750 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:40:04,098 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:40:04,751 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:40:08,579 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 02:40:08,579 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 02:40:10,732 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:40:14,099 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:40:15,733 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:40:19,109 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:40:20,125 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 02:40:21,132 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:40:22,837 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 02:40:22,838 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 02:40:22,838 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 02:40:22,841 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 02:40:23,579 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 02:40:23,579 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 02:40:23,719 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 02:40:25,721 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 02:40:26,676 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:40:29,252 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:40:29,253 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:40:31,677 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:40:34,253 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:40:34,254 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:40:36,678 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:40:38,579 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 02:40:38,579 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 02:40:41,734 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:40:44,255 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:40:44,255 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:40:46,735 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:40:49,267 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:40:50,137 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 02:40:52,138 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:40:52,720 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 02:40:52,722 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 02:40:52,722 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 02:40:52,724 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 02:40:52,747 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 02:40:53,579 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 02:40:53,580 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 02:40:55,748 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 02:40:57,657 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:40:59,264 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:40:59,264 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:41:02,658 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:41:04,265 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:41:04,265 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:41:07,659 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:41:08,579 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 02:41:08,579 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 02:41:12,738 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:41:14,266 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:41:14,266 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:41:17,744 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:41:19,276 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:41:20,137 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 02:41:22,211 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 02:41:22,213 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 02:41:22,213 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 02:41:22,215 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 02:41:22,775 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 02:41:23,216 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:41:23,579 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 02:41:23,580 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 02:41:23,776 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 02:41:28,656 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:41:28,656 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:41:28,715 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:41:33,656 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:41:33,656 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:41:33,716 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:41:38,579 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 02:41:38,580 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 02:41:38,748 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:41:43,658 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:41:43,658 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:41:43,749 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:41:48,669 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:41:48,752 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:41:50,137 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 02:41:53,587 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 02:41:53,588 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 02:41:53,999 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 02:41:54,001 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 02:41:54,002 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 02:41:54,002 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:41:54,003 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 02:41:54,804 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 02:41:55,805 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 02:41:58,665 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:41:59,005 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:42:03,666 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:42:04,005 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:42:08,583 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 02:42:08,584 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 02:42:09,656 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:42:13,668 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:42:14,656 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:42:18,668 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:42:19,657 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:42:20,141 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 02:42:23,590 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 02:42:23,590 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 02:42:24,743 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:42:26,963 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 02:42:26,965 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 02:42:26,966 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 02:42:26,969 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 02:42:27,838 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 02:42:28,678 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:42:29,839 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 02:42:29,971 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:42:33,679 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:42:34,971 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:42:38,590 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 02:42:38,591 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 02:42:40,714 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:42:43,823 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:42:43,823 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:42:45,714 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:42:48,824 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:42:48,824 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:42:50,146 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 02:42:51,148 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:42:53,590 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 02:42:53,591 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 02:42:56,758 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:42:58,842 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:43:00,311 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 02:43:00,312 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 02:43:00,313 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 02:43:00,314 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 02:43:00,872 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 02:43:01,873 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 02:43:02,315 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:43:03,835 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:43:07,315 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:43:08,590 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 02:43:08,590 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 02:43:12,680 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:43:13,837 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:43:17,681 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:43:18,838 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:43:20,149 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 02:43:23,151 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:43:23,590 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 02:43:23,591 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 02:43:28,740 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:43:28,846 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:43:31,722 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 02:43:31,723 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 02:43:31,724 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 02:43:31,725 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 02:43:31,901 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 02:43:33,902 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 02:43:34,161 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:43:34,161 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:43:34,726 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:43:38,590 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 02:43:38,591 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 02:43:39,744 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:43:44,162 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:43:44,163 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:43:44,745 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:43:49,163 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:43:49,164 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:43:49,746 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:43:50,153 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 02:43:53,591 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 02:43:53,591 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 02:43:55,665 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:43:59,168 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:44:00,674 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:44:04,172 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:44:04,322 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 02:44:04,324 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 02:44:04,324 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 02:44:04,325 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 02:44:04,939 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 02:44:05,940 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 02:44:06,326 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:44:08,591 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 02:44:08,591 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 02:44:11,712 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:44:14,174 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:44:16,712 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:44:19,396 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:44:19,396 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:44:20,157 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 02:44:22,159 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:44:23,591 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 02:44:23,592 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 02:44:27,696 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:44:29,404 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:44:32,710 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:44:34,106 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 02:44:34,108 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 02:44:34,108 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 02:44:34,110 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 02:44:34,509 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:44:34,509 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:44:34,968 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 02:44:35,969 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 02:44:38,111 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:44:38,591 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 02:44:38,592 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 02:44:43,765 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:44:44,510 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:44:44,511 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:44:48,766 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:44:49,734 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:44:49,735 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:44:50,161 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 02:44:53,591 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 02:44:53,592 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 02:44:54,745 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:44:58,745 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:44:59,750 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:45:03,743 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:45:03,750 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:45:04,121 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 02:45:04,122 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 02:45:04,122 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 02:45:04,124 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 02:45:04,996 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 02:45:05,124 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:45:05,997 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 02:45:08,591 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 02:45:08,592 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 02:45:10,684 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:45:13,748 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:45:13,748 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:45:15,685 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:45:18,749 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:45:18,749 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:45:20,166 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 02:45:21,168 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:45:23,591 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 02:45:23,592 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 02:45:26,666 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:45:28,754 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:45:31,670 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:45:32,316 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 02:45:32,318 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 02:45:32,319 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 02:45:32,320 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 02:45:33,029 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 02:45:33,757 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:45:34,030 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 02:45:37,321 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:45:38,591 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 02:45:38,592 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 02:45:42,716 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:45:43,758 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:45:47,717 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:45:48,759 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:45:50,169 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 02:45:53,171 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:45:53,592 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 02:45:53,592 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 02:45:58,728 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:45:58,767 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:46:01,482 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 02:46:01,483 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 02:46:01,484 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 02:46:01,485 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 02:46:02,057 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 02:46:03,881 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:46:03,881 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:46:04,059 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 02:46:04,486 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:46:08,592 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 02:46:08,592 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 02:46:09,700 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:46:13,883 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:46:13,883 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:46:14,701 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:46:18,884 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:46:19,701 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:46:20,173 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 02:46:23,592 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 02:46:23,592 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 02:46:24,761 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:46:28,892 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:46:29,767 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:46:31,498 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 02:46:31,499 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 02:46:31,500 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 02:46:31,501 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 02:46:32,084 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 02:46:33,901 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:46:33,902 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:46:34,086 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 02:46:35,502 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:46:38,592 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 02:46:38,592 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 02:46:40,698 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:46:43,903 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:46:43,903 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:46:45,699 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:46:48,904 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:46:48,904 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:46:50,177 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 02:46:51,179 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:46:53,592 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 02:46:53,593 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 02:46:56,761 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:46:58,914 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:47:01,771 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:47:03,910 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:47:04,062 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 02:47:04,064 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 02:47:04,064 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 02:47:04,065 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 02:47:04,107 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 02:47:06,109 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 02:47:07,066 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:47:08,592 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 02:47:08,593 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 02:47:12,665 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:47:13,911 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:47:17,666 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:47:18,912 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:47:20,181 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 02:47:23,183 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:47:23,592 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 02:47:23,593 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 02:47:28,766 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:47:28,919 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:47:33,016 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 02:47:33,018 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 02:47:33,018 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 02:47:33,020 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 02:47:33,145 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 02:47:33,920 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:47:34,021 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:47:34,146 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 02:47:38,592 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 02:47:38,593 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 02:47:39,700 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:47:43,922 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:47:44,700 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:47:49,146 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:47:49,146 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:47:49,701 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:47:50,185 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 02:47:53,592 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 02:47:53,593 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 02:47:54,733 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:47:59,153 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:47:59,742 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:48:02,436 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 02:48:02,438 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 02:48:02,438 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 02:48:02,439 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 02:48:03,170 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 02:48:04,155 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:48:04,156 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:48:04,172 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 02:48:05,440 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:48:08,593 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 02:48:08,593 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 02:48:10,761 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:48:14,157 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:48:14,157 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:48:15,762 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:48:19,158 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:48:19,158 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:48:20,190 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 02:48:21,192 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:48:23,593 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 02:48:23,593 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 02:48:26,723 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:48:29,168 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:48:31,732 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:48:33,413 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 02:48:33,414 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 02:48:33,415 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 02:48:33,416 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 02:48:34,166 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:48:34,198 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 02:48:34,199 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 02:48:37,417 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:48:38,593 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 02:48:38,593 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 02:48:42,667 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:48:44,168 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:48:47,667 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:48:49,168 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:48:50,193 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 02:48:53,195 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:48:53,593 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 02:48:53,593 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 02:48:58,721 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:48:59,178 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:49:01,236 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 02:49:01,238 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 02:49:01,238 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 02:49:01,239 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 02:49:02,221 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 02:49:02,222 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 02:49:04,176 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:49:04,240 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:49:08,593 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 02:49:08,594 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 02:49:09,677 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:49:14,178 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:49:14,678 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:49:19,178 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:49:19,679 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:49:20,197 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 02:49:23,609 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 02:49:23,615 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 02:49:24,758 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:49:29,184 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:49:29,202 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 02:49:29,203 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 02:49:29,203 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 02:49:29,205 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 02:49:29,249 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 02:49:30,206 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:49:30,250 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 02:49:34,185 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:49:35,207 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:49:38,600 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 02:49:38,601 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 02:49:40,736 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:49:44,187 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:49:45,737 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:49:49,188 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:49:50,201 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 02:49:51,203 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:49:53,600 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 02:49:53,601 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 02:49:56,681 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:49:59,195 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:49:59,422 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 02:49:59,424 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 02:49:59,425 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 02:49:59,427 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 02:50:00,274 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 02:50:00,274 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 02:50:02,428 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:50:04,196 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:50:07,428 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:50:08,601 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 02:50:08,601 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 02:50:12,732 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:50:14,198 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:50:17,732 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:50:19,199 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:50:20,210 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 02:50:23,217 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:50:23,606 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 02:50:23,607 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 02:50:25,829 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 02:50:25,830 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 02:50:25,831 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 02:50:25,833 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 02:50:26,295 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 02:50:28,296 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 02:50:28,834 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:50:29,229 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:50:33,834 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:50:34,230 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:50:38,604 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 02:50:38,605 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 02:50:39,771 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:50:44,232 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:50:44,772 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:50:49,232 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:50:49,773 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:50:50,212 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 02:50:53,604 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 02:50:53,605 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 02:50:55,753 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:50:59,238 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:50:59,635 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 02:50:59,637 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 02:50:59,638 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 02:50:59,640 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 02:51:00,324 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 02:51:00,325 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 02:51:01,641 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:51:04,239 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:51:06,641 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:51:08,604 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 02:51:08,605 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 02:51:11,724 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:51:14,240 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:51:16,724 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:51:19,241 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:51:20,216 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 02:51:22,218 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:51:23,604 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 02:51:23,605 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 02:51:27,730 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:51:29,250 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:51:30,948 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 02:51:30,951 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 02:51:30,951 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 02:51:30,953 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 02:51:31,351 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 02:51:32,353 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 02:51:32,954 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:51:34,350 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:51:37,954 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:51:38,605 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 02:51:38,605 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 02:51:43,709 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:51:44,352 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:51:48,710 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:51:49,498 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:51:49,498 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:51:50,220 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 02:51:53,605 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 02:51:53,605 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 02:51:54,687 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:51:59,505 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:51:59,505 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:51:59,694 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:52:02,946 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 02:52:02,948 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 02:52:02,948 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 02:52:02,950 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 02:52:03,378 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 02:52:04,379 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 02:52:04,506 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:52:04,506 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:52:04,951 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:52:08,605 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 02:52:08,606 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 02:52:10,787 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:52:14,507 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:52:15,788 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:52:19,597 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:52:20,224 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 02:52:21,226 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:52:23,605 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 02:52:23,605 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 02:52:26,775 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:52:29,607 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:52:31,788 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:52:33,017 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 02:52:33,019 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 02:52:33,019 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 02:52:33,021 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 02:52:33,404 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 02:52:34,405 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 02:52:34,606 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:52:34,606 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:52:37,022 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:52:38,605 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 02:52:38,605 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 02:52:42,772 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:52:43,607 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:52:47,773 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:52:48,614 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:52:50,227 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 02:52:53,230 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:52:53,605 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 02:52:53,606 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 02:52:58,626 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:52:58,699 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:53:00,405 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 02:53:00,407 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 02:53:00,407 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 02:53:00,409 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 02:53:00,431 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 02:53:02,433 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 02:53:03,817 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:53:04,410 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:53:08,605 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 02:53:08,606 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 02:53:09,695 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:53:13,819 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:53:14,696 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:53:18,819 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:53:19,697 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:53:20,231 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 02:53:23,605 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 02:53:23,606 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 02:53:24,713 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:53:28,831 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:53:29,717 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:53:30,434 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 02:53:30,435 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 02:53:30,436 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 02:53:30,438 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 02:53:30,460 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 02:53:32,463 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 02:53:33,870 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:53:33,870 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:53:35,439 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:53:38,605 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 02:53:38,606 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 02:53:40,710 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:53:43,871 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:53:43,872 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:53:45,711 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:53:48,872 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:53:48,872 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:53:50,235 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 02:53:51,238 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:53:53,606 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 02:53:53,606 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 02:53:56,783 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:53:58,888 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:54:00,534 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 02:54:00,535 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 02:54:00,536 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 02:54:00,537 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 02:54:01,490 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 02:54:02,491 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 02:54:02,538 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:54:03,944 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:54:03,944 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:54:07,539 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:54:08,606 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 02:54:08,606 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 02:54:12,762 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:54:13,945 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:54:13,946 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:54:17,763 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:54:18,946 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:54:18,947 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:54:20,239 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 02:54:23,242 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:54:23,606 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 02:54:23,607 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 02:54:28,761 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:54:28,957 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:54:29,404 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 02:54:29,405 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 02:54:29,406 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 02:54:29,408 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 02:54:29,530 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 02:54:30,531 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 02:54:33,958 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:54:34,409 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:54:38,606 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 02:54:38,607 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 02:54:39,732 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:54:43,959 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:54:44,732 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:54:48,960 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:54:49,733 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:54:50,243 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 02:54:53,606 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 02:54:53,607 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 02:54:55,724 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:54:58,968 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:54:59,703 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 02:54:59,704 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 02:54:59,704 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 02:54:59,706 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 02:55:00,556 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 02:55:00,556 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 02:55:01,706 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:55:04,104 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:55:06,707 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:55:08,606 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 02:55:08,607 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 02:55:12,698 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:55:14,105 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:55:14,105 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:55:17,699 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:55:19,106 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:55:19,106 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:55:20,247 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 02:55:23,250 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:55:23,618 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 02:55:23,625 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 02:55:28,750 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:55:29,114 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:55:29,115 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:55:29,186 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 02:55:29,187 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 02:55:29,187 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 02:55:29,189 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 02:55:29,591 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 02:55:30,592 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 02:55:34,115 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:55:34,190 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:55:38,609 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 02:55:38,610 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 02:55:39,705 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:55:44,117 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:55:44,117 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:55:44,706 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:55:49,118 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:55:49,118 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:55:49,706 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:55:50,249 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 02:55:53,614 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 02:55:53,615 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 02:55:54,785 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:55:59,123 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:55:59,472 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 02:55:59,474 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 02:55:59,474 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 02:55:59,476 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 02:55:59,618 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 02:56:00,477 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:56:00,619 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 02:56:04,124 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:56:05,478 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:56:08,612 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 02:56:08,613 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 02:56:10,765 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:56:14,125 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:56:15,765 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:56:19,126 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:56:20,254 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 02:56:21,256 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:56:23,612 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 02:56:23,613 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 02:56:26,790 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:56:29,135 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:56:31,795 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:56:32,021 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 02:56:32,022 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 02:56:32,023 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 02:56:32,024 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 02:56:32,646 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 02:56:32,647 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 02:56:34,174 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:56:37,026 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:56:38,612 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 02:56:38,613 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 02:56:42,730 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:56:44,175 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:56:47,731 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:56:49,176 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:56:50,257 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 02:56:53,260 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:56:53,612 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 02:56:53,613 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 02:56:58,752 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:56:59,183 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:57:03,647 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 02:57:03,649 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 02:57:03,650 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 02:57:03,651 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 02:57:03,681 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 02:57:04,185 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:57:04,652 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:57:04,683 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 02:57:08,612 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 02:57:08,613 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 02:57:09,750 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:57:14,187 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:57:14,750 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:57:19,261 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:57:19,261 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:57:19,751 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:57:20,261 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 02:57:23,613 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 02:57:23,613 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 02:57:25,728 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:57:29,271 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:57:30,733 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:57:33,723 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 02:57:33,725 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 02:57:33,726 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 02:57:33,727 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 02:57:34,272 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:57:34,707 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 02:57:34,707 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 02:57:36,728 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:57:38,613 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 02:57:38,613 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 02:57:42,688 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:57:44,273 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:57:47,689 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:57:49,274 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:57:50,265 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 02:57:53,268 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:57:53,613 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 02:57:53,613 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 02:57:58,747 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:57:59,285 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:58:02,715 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 02:58:02,716 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 02:58:02,717 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 02:58:02,718 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 02:58:02,727 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 02:58:04,284 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:58:04,719 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:58:04,729 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 02:58:08,613 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 02:58:08,613 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 02:58:10,705 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:58:14,285 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:58:15,706 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:58:19,286 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:58:20,269 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 02:58:21,272 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:58:23,613 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 02:58:23,614 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 02:58:26,699 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:58:29,294 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:58:31,715 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:58:32,984 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 02:58:32,986 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 02:58:32,986 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 02:58:32,988 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 02:58:33,757 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 02:58:34,384 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:58:34,758 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 02:58:36,989 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:58:38,613 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 02:58:38,614 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 02:58:42,714 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:58:44,385 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:58:47,715 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:58:49,386 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:58:49,387 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:58:50,273 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 02:58:53,276 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:58:53,613 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 02:58:53,614 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 02:58:58,743 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:58:59,396 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:59:02,391 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 02:59:02,393 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 02:59:02,393 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 02:59:02,395 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 02:59:02,789 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 02:59:02,789 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 02:59:04,395 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:59:04,396 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:59:08,613 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 02:59:08,614 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 02:59:09,743 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:59:14,397 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:59:14,743 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:59:19,523 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:59:19,744 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:59:20,278 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 02:59:23,613 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 02:59:23,614 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 02:59:25,729 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:59:29,524 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:59:30,729 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:59:34,529 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:59:35,733 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:59:36,672 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 02:59:36,674 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 02:59:36,674 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 02:59:36,675 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 02:59:36,817 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 02:59:38,614 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 02:59:38,614 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 02:59:38,821 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 02:59:40,769 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:59:43,624 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:59:45,769 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:59:48,625 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 02:59:50,280 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 02:59:51,282 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:59:53,614 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 02:59:53,614 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 02:59:56,787 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 02:59:58,631 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:00:01,790 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:00:02,128 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 03:00:02,130 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 03:00:02,131 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 03:00:02,132 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 03:00:02,848 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 03:00:02,848 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 03:00:03,634 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:00:07,134 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:00:08,614 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 03:00:08,614 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 03:00:12,770 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:00:13,636 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:00:17,771 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:00:18,689 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:00:20,284 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 03:00:23,287 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:00:23,614 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 03:00:23,614 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 03:00:28,698 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:00:28,788 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:00:31,615 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 03:00:31,617 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 03:00:31,618 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 03:00:31,619 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 03:00:31,868 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 03:00:32,869 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 03:00:33,763 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:00:34,620 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:00:38,614 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 03:00:38,615 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 03:00:39,766 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:00:43,891 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:00:44,767 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:00:48,892 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:00:49,767 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:00:50,288 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 03:00:53,614 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 03:00:53,615 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 03:00:55,750 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:00:58,904 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:01:00,752 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:01:00,965 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 03:01:00,967 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 03:01:00,967 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 03:01:00,969 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 03:01:01,888 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 03:01:02,889 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 03:01:03,899 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:01:05,970 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:01:08,614 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 03:01:08,615 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 03:01:11,691 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:01:13,900 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:01:16,692 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:01:18,901 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:01:20,291 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 03:01:22,293 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:01:23,614 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 03:01:23,615 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 03:01:27,743 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:01:28,619 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:01:32,028 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 03:01:32,030 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 03:01:32,031 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 03:01:32,032 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 03:01:32,920 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 03:01:32,920 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 03:01:33,033 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:01:33,622 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:01:38,034 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:01:38,615 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 03:01:38,615 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 03:01:43,623 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:01:43,740 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:01:48,648 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:01:48,741 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:01:50,296 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 03:01:53,615 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 03:01:53,616 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 03:01:53,784 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:01:58,658 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:01:58,790 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:02:01,065 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 03:02:01,067 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 03:02:01,067 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 03:02:01,069 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 03:02:01,947 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 03:02:02,949 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 03:02:03,655 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:02:04,070 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:02:08,615 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 03:02:08,615 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 03:02:09,704 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:02:13,696 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:02:14,705 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:02:18,697 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:02:19,705 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:02:20,299 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 03:02:23,615 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 03:02:23,615 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 03:02:25,702 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:02:28,708 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:02:30,035 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 03:02:30,036 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 03:02:30,036 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 03:02:30,037 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 03:02:30,980 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 03:02:30,980 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 03:02:31,038 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:02:33,706 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:02:36,038 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:02:38,615 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 03:02:38,616 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 03:02:41,776 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:02:43,708 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:02:46,777 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:02:48,708 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:02:50,303 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 03:02:52,305 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:02:53,615 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 03:02:53,616 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 03:02:57,703 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:02:58,717 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:03:00,315 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 03:03:00,317 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 03:03:00,317 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 03:03:00,319 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 03:03:01,015 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 03:03:02,016 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 03:03:03,320 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:03:03,721 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:03:08,321 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:03:08,615 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 03:03:08,616 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 03:03:13,722 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:03:13,766 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:03:18,723 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:03:18,766 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:03:20,307 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 03:03:23,616 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 03:03:23,616 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 03:03:24,694 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:03:28,740 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:03:29,698 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:03:29,913 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 03:03:29,915 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 03:03:29,915 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 03:03:29,917 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 03:03:30,053 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 03:03:32,055 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 03:03:33,737 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:03:34,918 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:03:38,616 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 03:03:38,616 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 03:03:40,794 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:03:43,890 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:03:45,795 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:03:48,891 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:03:50,311 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 03:03:51,313 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:03:53,616 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 03:03:53,616 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 03:03:56,756 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:03:58,909 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:04:00,521 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 03:04:00,522 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 03:04:00,522 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 03:04:00,523 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 03:04:01,081 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 03:04:02,082 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 03:04:02,524 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:04:03,928 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:04:07,525 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:04:08,616 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 03:04:08,616 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 03:04:12,764 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:04:13,929 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:04:17,765 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:04:18,930 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:04:20,315 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 03:04:23,317 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:04:23,616 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 03:04:23,616 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 03:04:28,753 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:04:28,942 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:04:31,386 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 03:04:31,387 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 03:04:31,387 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 03:04:31,388 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 03:04:32,114 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 03:04:33,941 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:04:34,116 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 03:04:34,389 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:04:38,616 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 03:04:38,617 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 03:04:39,746 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:04:43,942 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:04:44,747 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:04:48,943 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:04:49,747 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:04:50,319 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 03:04:53,616 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 03:04:53,617 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 03:04:55,725 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:04:58,957 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:05:00,067 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 03:05:00,068 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 03:05:00,069 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 03:05:00,071 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 03:05:00,137 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 03:05:01,072 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:05:02,139 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 03:05:03,954 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:05:06,072 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:05:08,616 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 03:05:08,617 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 03:05:11,739 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:05:13,956 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:05:16,740 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:05:18,957 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:05:20,323 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 03:05:22,325 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:05:23,616 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 03:05:23,617 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 03:05:27,744 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:05:28,964 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:05:32,104 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 03:05:32,105 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 03:05:32,105 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 03:05:32,106 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 03:05:32,168 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 03:05:33,107 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:05:33,964 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:05:34,170 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 03:05:38,108 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:05:38,616 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 03:05:38,617 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 03:05:43,749 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:05:43,965 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:05:48,750 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:05:49,084 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:05:50,327 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 03:05:53,617 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 03:05:53,617 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 03:05:53,761 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:05:58,770 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:05:59,096 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:06:02,557 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 03:06:02,558 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 03:06:02,559 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 03:06:02,560 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 03:06:03,196 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 03:06:04,091 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:06:04,197 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 03:06:04,561 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:06:08,617 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 03:06:08,617 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 03:06:09,725 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:06:14,092 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:06:14,726 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:06:19,093 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:06:19,726 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:06:20,331 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 03:06:23,617 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 03:06:23,617 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 03:06:25,693 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:06:29,104 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:06:30,696 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:06:34,103 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:06:34,621 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 03:06:34,623 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 03:06:34,623 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 03:06:34,625 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 03:06:35,222 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 03:06:36,223 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 03:06:36,626 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:06:38,617 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 03:06:38,618 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 03:06:41,739 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:06:44,105 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:06:46,740 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:06:49,106 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:06:50,335 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 03:06:52,337 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:06:53,617 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 03:06:53,617 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 03:06:57,735 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:06:59,107 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:07:02,741 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:07:04,114 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:07:05,458 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 03:07:05,460 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 03:07:05,461 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 03:07:05,462 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 03:07:06,251 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 03:07:08,253 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 03:07:08,463 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:07:08,617 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 03:07:08,618 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 03:07:13,781 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:07:14,114 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:07:18,782 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:07:19,115 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:07:20,339 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 03:07:23,617 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 03:07:23,618 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 03:07:24,752 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:07:29,116 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:07:29,752 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:07:34,130 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:07:34,757 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:07:36,788 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 03:07:36,789 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 03:07:36,789 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 03:07:36,791 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 03:07:37,292 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 03:07:38,293 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 03:07:38,617 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 03:07:38,618 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 03:07:39,773 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:07:44,194 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:07:44,774 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:07:49,195 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:07:49,774 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:07:50,342 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 03:07:53,618 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 03:07:53,618 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 03:07:54,782 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:07:58,659 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:07:59,782 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:08:03,660 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:08:04,788 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:08:08,620 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 03:08:08,620 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 03:08:09,238 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 03:08:09,240 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 03:08:09,241 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 03:08:09,242 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 03:08:09,321 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 03:08:10,243 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:08:12,323 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 03:08:13,669 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:08:15,243 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:08:18,670 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:08:20,244 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:08:20,346 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 03:08:23,620 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 03:08:23,621 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 03:08:25,773 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:08:28,846 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:08:30,774 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:08:33,854 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:08:35,782 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:08:38,195 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 03:08:38,196 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 03:08:38,197 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 03:08:38,198 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 03:08:38,353 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 03:08:38,620 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 03:08:38,621 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 03:08:40,354 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 03:08:41,777 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:08:43,854 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:08:46,777 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:08:48,855 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:08:50,350 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 03:08:52,353 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:08:53,620 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 03:08:53,621 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 03:08:57,751 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:08:58,856 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:09:02,760 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:09:03,865 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:09:05,421 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 03:09:05,422 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 03:09:05,423 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 03:09:05,424 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 03:09:06,378 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 03:09:08,379 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 03:09:08,425 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:09:08,620 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 03:09:08,621 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 03:09:13,706 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:09:13,862 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:09:18,707 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:09:18,862 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:09:20,354 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 03:09:23,621 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 03:09:23,621 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 03:09:23,733 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:09:28,733 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:09:28,864 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:09:33,739 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:09:33,870 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:09:38,348 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 03:09:38,349 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 03:09:38,350 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 03:09:38,351 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 03:09:38,405 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 03:09:38,755 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 03:09:38,755 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 03:09:38,885 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:09:40,406 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 03:09:43,881 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:09:43,886 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:09:48,882 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:09:48,887 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:09:50,359 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 03:09:53,752 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 03:09:53,752 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 03:09:53,921 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:09:58,883 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:09:58,922 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:10:03,890 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:10:03,924 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:10:08,581 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 03:10:08,582 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 03:10:08,583 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 03:10:08,584 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 03:10:08,976 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:10:08,977 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 03:10:08,978 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 03:10:09,077 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:10:09,432 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 03:10:10,433 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 03:10:14,078 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:10:14,503 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:10:19,079 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:10:19,504 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:10:20,360 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 03:10:23,976 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 03:10:23,977 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 03:10:24,146 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:10:28,978 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:10:29,146 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:10:33,986 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:10:34,153 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:10:38,651 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 03:10:38,652 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 03:10:38,652 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 03:10:38,657 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 03:10:39,035 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:10:39,036 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 03:10:39,037 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 03:10:39,193 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:10:39,460 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 03:10:40,461 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 03:10:44,194 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:10:44,608 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:10:49,195 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:10:49,609 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:10:50,364 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 03:10:54,035 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 03:10:54,036 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 03:10:55,163 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:10:59,038 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:11:00,164 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:11:04,038 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:11:05,165 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:11:09,047 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 03:11:09,048 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 03:11:10,209 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:11:11,776 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 03:11:11,777 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 03:11:11,777 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 03:11:11,780 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 03:11:12,490 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 03:11:14,492 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 03:11:14,817 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:11:14,817 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:11:15,781 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:11:19,818 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:11:20,368 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 03:11:21,370 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:11:24,042 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 03:11:24,043 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 03:11:27,148 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:11:29,820 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:11:29,820 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:11:32,149 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:11:34,821 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:11:34,821 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:11:37,158 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:11:39,050 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 03:11:39,052 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 03:11:42,169 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:11:43,054 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 03:11:43,056 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 03:11:43,056 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 03:11:43,061 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 03:11:43,521 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 03:11:44,522 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 03:11:44,830 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:11:44,831 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:11:48,062 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:11:49,831 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:11:49,832 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:11:50,371 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 03:11:53,373 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:11:54,048 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 03:11:54,049 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 03:11:59,217 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:11:59,833 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:11:59,833 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:12:04,218 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:12:04,834 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:12:04,834 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:12:09,052 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 03:12:09,053 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 03:12:10,212 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:12:13,103 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 03:12:13,105 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 03:12:13,105 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 03:12:13,111 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 03:12:13,550 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 03:12:14,551 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 03:12:14,841 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:12:16,112 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:12:19,842 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:12:19,843 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:12:20,375 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 03:12:21,377 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:12:24,051 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 03:12:24,052 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 03:12:27,224 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:12:29,844 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:12:32,225 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:12:34,844 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:12:37,226 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:12:39,056 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 03:12:39,062 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 03:12:43,208 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:12:43,474 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 03:12:43,476 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 03:12:43,477 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 03:12:43,483 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 03:12:43,584 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 03:12:44,586 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 03:12:44,873 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:12:44,873 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:12:48,484 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:12:49,874 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:12:49,874 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:12:50,380 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 03:12:54,053 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 03:12:54,054 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 03:12:54,206 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:12:59,206 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:12:59,875 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:13:04,207 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:13:04,876 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:13:09,066 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 03:13:09,070 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 03:13:10,145 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:13:14,202 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 03:13:14,204 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 03:13:14,205 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 03:13:14,209 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 03:13:14,618 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 03:13:14,880 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:13:15,209 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:13:16,620 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 03:13:19,881 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:13:20,210 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:13:20,383 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 03:13:24,059 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 03:13:24,060 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 03:13:25,226 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:13:29,882 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:13:30,227 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:13:34,883 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:13:35,228 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:13:39,063 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 03:13:39,064 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 03:13:41,210 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:13:44,833 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 03:13:44,834 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 03:13:44,834 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 03:13:44,837 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 03:13:45,237 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:13:45,238 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:13:45,642 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 03:13:46,643 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 03:13:46,838 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:13:50,239 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:13:50,387 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 03:13:52,389 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:13:54,062 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 03:13:54,063 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 03:13:58,183 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:13:59,337 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:13:59,338 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:14:03,183 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:14:04,339 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:14:04,339 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:14:08,190 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:14:09,070 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 03:14:09,071 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 03:14:12,025 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 03:14:12,026 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 03:14:12,027 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 03:14:12,029 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 03:14:12,674 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 03:14:14,030 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:14:14,676 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 03:14:14,916 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:14:14,916 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:14:19,030 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:14:19,917 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:14:19,917 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:14:20,390 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 03:14:24,068 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 03:14:24,068 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 03:14:24,155 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:14:29,156 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:14:29,919 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:14:29,919 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:14:34,156 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:14:34,920 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:14:34,920 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:14:39,075 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 03:14:39,076 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 03:14:39,169 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:14:43,057 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 03:14:43,058 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 03:14:43,058 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 03:14:43,059 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 03:14:43,700 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 03:14:44,701 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 03:14:44,924 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:14:45,060 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:14:49,925 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:14:50,061 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:14:50,394 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 03:14:54,072 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 03:14:54,073 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 03:14:55,216 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:14:59,926 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:15:00,217 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:15:04,927 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:15:05,218 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:15:09,076 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 03:15:09,079 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 03:15:10,230 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:15:14,427 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 03:15:14,428 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 03:15:14,429 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 03:15:14,430 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 03:15:14,733 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 03:15:14,932 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:15:15,431 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:15:16,734 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 03:15:19,932 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:15:20,399 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 03:15:21,401 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:15:24,076 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 03:15:24,077 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 03:15:27,198 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:15:29,943 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:15:32,198 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:15:34,943 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:15:37,199 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:15:39,076 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 03:15:39,077 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 03:15:42,229 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:15:44,953 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:15:47,230 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:15:47,712 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 03:15:47,713 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 03:15:47,713 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 03:15:47,714 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 03:15:47,769 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 03:15:48,770 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 03:15:50,117 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:15:50,402 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 03:15:52,405 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:15:54,076 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 03:15:54,076 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 03:15:58,158 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:15:59,119 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:15:59,119 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:16:03,159 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:16:04,119 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:16:04,120 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:16:08,160 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:16:09,077 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 03:16:09,077 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 03:16:13,182 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:16:14,121 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:16:14,121 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:16:18,187 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:16:19,132 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:16:20,129 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 03:16:20,130 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 03:16:20,131 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 03:16:20,132 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 03:16:20,407 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 03:16:20,807 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 03:16:22,809 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 03:16:23,409 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:16:24,077 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 03:16:24,077 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 03:16:29,132 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:16:29,132 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:16:29,158 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:16:34,133 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:16:34,133 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:16:34,158 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:16:39,077 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 03:16:39,077 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 03:16:39,213 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:16:44,145 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:16:44,218 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:16:49,140 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:16:49,219 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:16:49,776 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 03:16:49,778 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 03:16:49,778 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 03:16:49,779 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 03:16:49,839 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 03:16:50,411 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 03:16:50,840 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 03:16:54,077 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 03:16:54,077 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 03:16:55,190 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:16:59,193 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:17:00,190 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:17:04,194 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:17:04,194 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:17:05,191 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:17:09,077 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 03:17:09,077 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 03:17:11,178 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:17:14,195 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:17:14,195 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:17:16,179 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:17:19,206 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:17:20,414 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 03:17:21,415 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:17:22,269 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 03:17:22,271 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 03:17:22,271 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 03:17:22,273 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 03:17:22,870 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 03:17:24,077 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 03:17:24,078 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 03:17:24,872 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 03:17:27,199 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:17:29,203 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:17:32,200 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:17:34,204 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:17:37,200 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:17:39,077 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 03:17:39,078 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 03:17:42,230 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:17:44,205 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:17:47,240 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:17:49,213 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:17:50,417 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 03:17:52,418 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:17:52,480 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 03:17:52,481 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 03:17:52,481 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 03:17:52,483 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 03:17:52,899 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 03:17:54,077 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 03:17:54,078 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 03:17:54,902 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 03:17:58,232 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:17:59,213 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:18:03,233 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:18:04,214 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:18:08,233 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:18:09,077 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 03:18:09,078 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 03:18:14,178 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:18:14,215 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:18:19,186 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:18:19,224 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:18:19,225 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:18:20,423 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 03:18:22,042 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 03:18:22,043 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 03:18:22,043 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 03:18:22,045 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 03:18:22,929 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 03:18:22,929 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 03:18:24,078 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 03:18:24,078 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 03:18:25,150 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:18:29,450 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:18:30,151 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:18:34,451 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:18:34,451 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:18:35,151 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:18:39,078 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 03:18:39,078 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 03:18:40,209 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:18:44,452 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:18:44,452 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:18:45,209 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:18:49,459 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:18:50,216 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:18:50,429 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 03:18:53,908 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 03:18:53,910 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 03:18:53,911 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 03:18:53,912 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 03:18:53,952 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 03:18:54,311 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 03:18:54,312 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 03:18:54,953 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 03:18:55,436 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:18:59,462 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:19:00,437 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:19:04,463 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:19:05,438 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:19:09,311 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 03:19:09,311 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 03:19:10,457 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:19:14,544 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:19:15,457 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:19:19,544 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:19:20,445 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 03:19:21,446 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:19:24,317 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 03:19:24,318 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 03:19:25,652 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 03:19:25,653 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 03:19:25,653 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 03:19:25,655 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 03:19:25,986 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 03:19:26,655 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:19:26,987 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 03:19:29,552 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:19:31,656 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:19:34,552 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:19:36,657 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:19:39,317 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 03:19:39,318 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 03:19:42,431 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:19:44,553 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:19:47,431 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:19:49,554 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:19:50,434 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 03:19:52,439 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:19:54,324 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 03:19:54,325 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 03:19:55,545 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 03:19:55,546 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 03:19:55,547 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 03:19:55,552 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 03:19:56,012 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 03:19:57,013 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 03:19:57,553 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:19:59,693 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:20:02,554 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:20:04,693 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:20:04,694 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:20:07,555 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:20:09,321 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 03:20:09,322 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 03:20:13,395 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:20:14,695 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:20:14,695 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:20:18,396 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:20:19,704 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:20:20,448 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 03:20:23,462 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:20:24,241 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 03:20:24,243 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 03:20:24,243 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 03:20:24,245 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 03:20:24,638 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 03:20:24,639 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 03:20:25,042 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 03:20:25,042 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 03:20:28,751 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:20:29,706 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:20:33,751 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:20:34,707 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:20:38,752 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:20:39,388 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 03:20:39,389 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 03:20:44,525 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:20:44,895 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:20:44,895 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:20:49,534 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:20:49,902 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:20:50,450 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 03:20:51,885 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 03:20:51,887 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 03:20:51,887 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 03:20:51,889 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 03:20:52,063 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 03:20:53,064 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 03:20:54,388 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 03:20:54,389 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 03:20:54,536 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:20:59,536 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:21:00,032 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:21:00,032 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:21:04,537 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:21:05,033 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:21:05,033 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:21:09,388 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 03:21:09,389 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 03:21:09,548 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:21:14,549 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:21:15,034 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:21:19,554 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:21:20,045 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:21:20,449 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 03:21:22,683 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 03:21:22,684 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 03:21:22,685 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 03:21:22,687 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 03:21:23,093 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 03:21:23,093 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 03:21:24,389 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 03:21:24,389 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 03:21:25,456 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:21:30,109 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:21:30,456 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:21:35,110 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:21:35,457 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:21:39,389 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 03:21:39,389 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 03:21:40,481 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:21:45,111 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:21:45,482 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:21:50,125 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:21:50,456 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 03:21:51,459 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:21:52,835 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 03:21:52,836 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 03:21:52,836 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 03:21:52,838 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 03:21:53,120 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 03:21:53,121 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 03:21:54,389 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 03:21:54,389 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 03:21:56,473 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:22:00,238 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:22:01,473 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:22:05,387 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:22:05,388 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:22:06,474 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:22:09,389 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 03:22:09,389 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 03:22:11,478 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:22:15,389 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:22:15,389 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:22:16,481 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:22:20,399 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:22:20,459 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 03:22:21,958 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 03:22:21,960 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 03:22:21,960 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 03:22:21,960 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:22:21,962 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 03:22:22,153 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 03:22:23,154 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 03:22:24,389 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 03:22:24,390 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 03:22:27,486 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:22:29,396 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:22:32,487 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:22:34,397 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:22:37,488 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:22:39,389 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 03:22:39,390 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 03:22:42,490 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:22:44,398 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:22:47,493 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:22:49,400 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:22:50,470 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 03:22:53,336 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 03:22:53,338 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 03:22:53,339 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 03:22:53,339 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:22:53,341 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 03:22:54,182 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 03:22:54,389 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 03:22:54,390 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 03:22:55,183 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 03:22:58,552 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:22:59,483 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:23:03,553 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:23:04,483 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:23:08,554 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:23:09,389 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 03:23:09,390 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 03:23:14,472 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:23:14,485 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:23:19,478 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:23:19,488 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:23:20,463 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 03:23:24,392 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 03:23:24,392 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 03:23:25,162 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 03:23:25,164 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 03:23:25,164 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 03:23:25,166 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:23:25,168 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 03:23:25,207 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 03:23:27,209 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 03:23:29,567 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:23:29,567 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:23:30,169 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:23:34,568 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:23:34,568 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:23:35,169 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:23:39,392 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 03:23:39,393 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 03:23:40,553 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:23:44,569 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:23:44,570 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:23:45,553 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:23:49,570 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:23:49,571 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:23:50,472 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 03:23:51,476 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:23:54,398 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 03:23:54,402 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 03:23:55,545 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 03:23:55,547 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 03:23:55,547 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 03:23:55,552 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 03:23:56,233 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 03:23:56,553 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:23:57,235 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 03:23:59,693 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:23:59,694 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:24:01,554 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:24:04,694 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:24:06,554 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:24:09,396 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 03:24:09,397 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 03:24:12,491 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:24:14,695 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:24:17,491 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:24:19,703 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:24:20,466 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 03:24:23,467 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:24:24,396 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 03:24:24,396 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 03:24:25,059 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 03:24:25,061 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 03:24:25,062 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 03:24:25,063 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 03:24:25,263 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 03:24:25,263 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 03:24:29,064 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:24:29,704 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:24:34,065 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:24:34,704 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:24:34,705 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:24:39,066 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:24:39,396 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 03:24:39,397 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 03:24:44,559 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:24:44,706 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:24:44,706 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:24:49,567 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:24:49,709 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:24:50,471 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 03:24:54,397 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 03:24:54,397 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 03:24:54,516 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 03:24:54,553 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 03:24:54,554 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 03:24:54,555 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 03:24:55,291 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 03:24:55,291 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 03:24:55,556 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:24:59,955 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:25:00,557 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:25:04,956 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:25:04,956 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:25:05,558 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:25:09,397 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 03:25:09,397 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 03:25:10,573 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:25:14,958 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:25:14,958 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:25:15,574 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:25:19,970 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:25:20,479 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 03:25:21,482 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:25:24,284 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 03:25:24,285 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 03:25:24,285 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 03:25:24,287 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 03:25:24,318 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 03:25:24,694 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 03:25:24,695 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 03:25:25,319 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 03:25:26,803 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:25:29,967 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:25:31,803 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:25:34,967 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:25:36,804 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:25:39,430 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 03:25:39,430 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 03:25:42,584 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:25:44,969 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:25:47,585 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:25:49,970 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:25:49,970 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:25:50,487 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 03:25:53,494 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:25:54,445 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 03:25:54,446 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 03:25:56,082 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 03:25:56,083 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 03:25:56,083 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 03:25:56,085 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 03:25:56,352 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 03:25:57,354 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 03:25:59,086 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:25:59,978 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:25:59,978 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:26:04,087 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:26:04,979 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:26:04,979 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:26:09,088 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:26:09,435 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 03:26:09,436 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 03:26:14,509 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:26:14,981 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:26:14,981 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:26:19,510 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:26:19,993 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:26:20,488 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 03:26:24,435 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 03:26:24,435 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 03:26:24,563 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:26:25,404 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 03:26:25,405 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 03:26:25,405 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 03:26:25,408 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 03:26:26,383 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 03:26:27,385 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 03:26:29,989 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:26:30,409 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:26:34,990 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:26:35,410 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:26:39,435 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 03:26:39,436 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 03:26:40,934 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:26:45,208 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:26:45,209 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:26:45,935 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:26:50,218 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:26:50,491 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 03:26:51,492 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:26:53,304 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 03:26:53,306 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 03:26:53,306 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 03:26:53,308 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 03:26:53,409 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 03:26:54,410 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 03:26:54,435 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 03:26:54,436 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 03:26:56,596 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:27:00,218 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:27:01,596 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:27:05,219 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:27:05,219 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:27:06,597 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:27:09,436 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 03:27:09,436 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 03:27:12,584 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:27:15,220 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:27:17,585 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:27:20,228 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:27:20,507 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 03:27:23,509 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:27:24,441 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 03:27:24,442 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 03:27:25,489 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 03:27:25,693 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 03:27:25,694 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 03:27:25,695 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 03:27:26,439 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 03:27:28,441 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 03:27:28,696 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:27:30,227 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:27:33,697 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:27:35,228 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:27:38,698 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:27:39,439 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 03:27:39,440 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 03:27:44,515 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:27:45,337 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:27:49,520 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:27:50,346 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:27:50,503 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 03:27:54,440 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 03:27:54,441 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 03:27:54,546 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:27:54,561 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 03:27:54,563 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 03:27:54,563 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 03:27:54,565 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 03:27:55,468 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 03:27:56,469 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 03:27:59,566 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:28:00,345 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:28:04,567 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:28:05,345 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:28:09,439 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 03:28:09,440 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 03:28:09,570 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:28:14,571 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:28:15,347 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:28:19,578 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:28:20,354 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:28:20,505 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 03:28:23,520 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 03:28:23,522 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 03:28:23,523 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 03:28:23,525 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 03:28:24,440 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 03:28:24,441 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 03:28:24,502 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 03:28:24,587 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:28:26,504 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 03:28:29,589 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:28:30,354 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:28:34,589 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:28:35,355 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:28:39,440 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 03:28:39,440 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 03:28:40,548 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:28:45,357 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:28:45,548 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:28:50,358 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:28:50,511 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 03:28:51,513 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:28:54,444 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 03:28:54,444 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 03:28:56,487 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 03:28:56,489 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 03:28:56,490 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 03:28:56,495 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 03:28:56,530 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 03:28:57,496 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:28:58,531 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 03:29:00,365 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:29:02,497 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:29:05,365 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:29:07,497 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:29:09,443 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 03:29:09,444 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 03:29:12,527 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:29:14,495 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:29:14,496 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:29:17,528 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:29:19,496 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:29:19,496 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:29:20,524 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 03:29:22,528 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:29:24,449 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 03:29:24,449 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 03:29:25,881 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 03:29:25,883 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 03:29:25,884 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 03:29:25,885 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 03:29:26,558 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 03:29:27,886 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:29:28,559 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 03:29:29,504 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:29:32,886 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:29:34,505 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:29:37,887 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:29:39,449 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 03:29:39,450 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 03:29:43,591 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:29:44,506 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:29:48,591 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:29:49,507 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:29:50,518 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 03:29:54,454 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 03:29:54,461 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 03:29:54,608 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:29:56,410 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 03:29:56,411 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 03:29:56,411 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 03:29:56,412 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 03:29:56,583 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 03:29:58,585 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 03:29:59,557 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:29:59,558 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:30:00,413 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:30:04,558 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:30:05,414 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:30:09,451 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 03:30:09,452 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 03:30:10,571 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:30:14,560 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:30:14,560 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:30:15,572 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:30:19,561 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:30:19,561 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:30:20,532 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 03:30:21,533 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:30:24,458 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 03:30:24,459 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 03:30:25,733 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 03:30:25,734 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 03:30:25,735 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 03:30:25,736 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 03:30:26,608 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 03:30:26,737 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:30:28,609 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 03:30:29,569 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:30:31,738 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:30:34,570 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:30:36,739 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:30:39,491 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 03:30:39,492 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 03:30:42,651 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:30:44,571 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:30:47,651 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:30:49,572 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:30:50,528 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 03:30:53,541 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:30:54,489 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 03:30:54,489 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 03:30:56,242 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 03:30:56,244 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 03:30:56,244 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 03:30:56,246 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 03:30:56,641 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 03:30:58,642 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 03:30:59,247 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:30:59,652 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:31:04,248 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:31:04,653 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:31:09,249 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:31:09,489 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 03:31:09,490 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 03:31:14,604 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:31:14,655 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:31:19,612 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:31:19,657 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:31:20,533 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 03:31:22,814 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 03:31:22,816 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 03:31:22,817 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 03:31:22,819 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 03:31:23,670 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 03:31:24,490 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 03:31:24,490 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 03:31:24,619 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:31:24,672 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 03:31:29,620 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:31:29,661 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:31:34,620 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:31:34,662 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:31:39,490 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 03:31:39,490 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 03:31:39,690 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:31:44,663 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:31:44,691 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:31:49,676 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:31:49,693 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:31:50,542 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 03:31:53,797 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 03:31:53,799 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 03:31:53,799 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 03:31:53,800 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 03:31:54,528 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 03:31:54,529 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 03:31:54,698 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 03:31:54,700 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:31:56,700 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 03:31:59,671 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:31:59,700 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:32:04,672 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:32:04,701 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:32:09,528 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 03:32:09,529 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 03:32:10,656 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:32:14,674 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:32:15,656 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:32:19,680 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:32:20,541 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 03:32:21,549 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:32:24,528 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 03:32:24,528 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 03:32:24,896 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 03:32:24,898 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 03:32:24,899 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 03:32:24,904 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 03:32:25,723 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 03:32:26,724 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 03:32:26,905 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:32:29,685 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:32:31,905 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:32:34,685 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:32:36,906 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:32:39,528 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 03:32:39,529 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 03:32:42,689 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:32:44,687 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:32:47,690 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:32:49,692 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:32:50,547 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 03:32:53,552 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:32:54,531 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 03:32:54,532 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 03:32:55,048 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 03:32:55,050 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 03:32:55,050 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 03:32:55,053 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 03:32:55,749 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 03:32:56,750 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 03:32:59,055 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:32:59,691 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:33:04,055 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:33:04,691 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:33:09,056 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:33:09,532 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 03:33:09,532 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 03:33:14,693 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:33:14,700 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:33:19,693 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:33:19,701 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:33:20,545 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 03:33:24,537 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 03:33:24,541 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 03:33:24,719 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:33:28,406 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 03:33:28,407 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 03:33:28,407 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 03:33:28,410 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 03:33:28,776 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 03:33:29,801 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:33:30,410 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:33:30,778 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 03:33:34,802 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:33:34,802 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:33:35,411 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:33:39,535 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 03:33:39,535 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 03:33:40,667 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:33:44,940 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:33:44,940 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:33:45,668 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:33:49,941 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:33:49,941 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:33:50,549 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 03:33:51,551 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:33:54,543 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 03:33:54,549 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 03:33:56,719 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:33:58,361 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 03:33:58,362 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 03:33:58,362 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 03:33:58,364 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 03:33:58,806 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 03:33:59,952 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:33:59,953 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:34:00,808 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 03:34:02,365 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:34:04,954 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:34:07,365 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:34:09,540 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 03:34:09,540 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 03:34:12,713 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:34:14,955 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:34:17,713 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:34:19,964 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:34:19,964 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:34:20,557 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 03:34:23,561 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:34:24,544 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 03:34:24,549 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 03:34:25,755 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 03:34:25,757 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 03:34:25,758 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 03:34:25,759 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 03:34:25,831 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 03:34:26,832 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 03:34:28,761 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:34:30,164 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:34:30,165 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:34:33,761 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:34:35,165 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:34:38,762 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:34:39,541 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 03:34:39,542 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 03:34:44,702 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:34:45,166 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:34:49,709 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:34:50,177 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:34:50,552 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 03:34:53,845 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 03:34:53,847 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 03:34:53,847 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 03:34:53,849 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 03:34:53,859 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 03:34:54,544 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 03:34:54,544 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 03:34:54,861 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 03:34:55,705 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:35:00,248 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:35:00,248 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:35:00,705 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:35:05,249 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:35:05,249 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:35:05,706 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:35:09,542 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 03:35:09,542 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 03:35:11,661 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:35:15,251 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:35:15,251 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:35:16,661 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:35:20,251 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:35:20,252 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:35:20,558 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 03:35:22,564 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:35:24,550 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 03:35:24,556 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 03:35:27,708 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:35:27,993 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 03:35:27,995 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 03:35:27,996 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 03:35:27,998 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 03:35:28,897 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 03:35:28,898 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 03:35:30,438 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:35:32,999 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:35:35,438 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:35:38,000 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:35:39,545 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 03:35:39,545 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 03:35:43,712 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:35:45,440 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:35:48,713 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:35:50,440 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:35:50,562 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 03:35:54,546 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 03:35:54,546 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 03:35:54,663 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:35:59,442 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 03:35:59,443 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 03:35:59,444 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 03:35:59,445 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 03:35:59,588 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:35:59,924 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 03:36:00,446 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:36:00,925 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 03:36:04,879 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:36:05,447 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:36:09,546 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 03:36:09,547 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 03:36:10,640 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:36:14,881 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:36:15,641 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:36:19,881 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:36:20,566 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 03:36:21,569 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:36:24,546 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 03:36:24,547 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 03:36:26,686 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:36:29,883 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:36:31,692 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:36:34,888 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:36:35,410 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 03:36:35,413 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 03:36:35,413 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 03:36:35,415 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 03:36:35,970 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 03:36:36,972 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 03:36:37,416 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:36:39,546 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 03:36:39,547 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 03:36:42,725 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:36:44,815 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:36:47,726 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:36:49,847 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:36:50,571 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 03:36:53,573 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:36:54,546 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 03:36:54,547 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 03:36:58,685 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:36:59,861 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:37:03,707 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:37:04,858 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:37:05,591 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 03:37:05,593 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 03:37:05,593 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 03:37:05,595 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 03:37:06,001 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 03:37:07,003 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 03:37:09,547 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 03:37:09,548 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 03:37:09,719 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:37:14,719 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:37:14,988 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:37:19,720 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:37:19,989 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:37:19,989 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:37:20,574 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 03:37:24,546 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 03:37:24,547 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 03:37:25,719 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:37:29,994 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:37:30,724 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:37:34,698 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 03:37:34,699 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 03:37:34,699 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 03:37:34,701 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 03:37:35,029 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 03:37:35,106 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:37:35,107 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:37:36,701 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:37:37,031 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 03:37:39,547 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 03:37:39,547 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 03:37:42,659 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:37:45,108 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:37:45,108 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:37:47,660 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:37:50,109 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:37:50,109 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:37:50,578 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 03:37:53,581 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:37:54,547 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 03:37:54,547 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 03:37:58,657 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:38:00,111 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:38:00,111 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:38:03,665 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:38:05,120 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:38:06,775 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 03:38:06,777 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 03:38:06,777 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 03:38:06,778 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 03:38:07,058 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 03:38:08,779 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:38:09,060 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 03:38:09,547 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 03:38:09,548 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 03:38:14,660 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:38:15,202 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:38:15,202 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:38:19,660 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:38:20,203 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:38:20,203 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:38:20,582 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 03:38:24,547 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 03:38:24,548 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 03:38:25,658 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:38:30,204 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:38:30,204 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:38:30,659 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:38:35,210 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:38:35,664 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:38:39,232 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 03:38:39,234 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 03:38:39,234 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 03:38:39,236 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 03:38:39,682 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 03:38:39,683 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 03:38:40,096 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 03:38:40,786 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:38:41,098 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 03:38:45,209 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:38:45,786 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:38:50,210 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:38:50,586 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 03:38:51,589 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:38:54,681 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 03:38:54,682 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 03:38:56,756 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:39:00,287 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:39:01,757 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:39:05,295 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:39:06,762 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:39:09,038 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 03:39:09,039 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 03:39:09,040 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 03:39:09,041 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 03:39:09,123 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 03:39:09,682 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 03:39:09,682 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 03:39:11,125 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 03:39:11,771 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:39:15,436 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:39:15,436 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:39:16,771 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:39:20,437 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:39:20,437 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:39:20,590 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 03:39:22,593 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:39:24,681 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 03:39:24,682 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 03:39:27,757 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:39:30,439 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:39:30,439 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:39:32,758 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:39:35,452 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:39:37,766 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:39:39,682 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 03:39:39,683 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 03:39:39,899 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 03:39:39,901 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 03:39:39,902 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 03:39:39,903 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 03:39:40,161 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 03:39:41,162 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 03:39:42,905 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:39:45,448 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:39:47,905 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:39:50,449 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:39:50,595 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 03:39:53,598 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:39:54,682 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 03:39:54,682 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 03:39:58,809 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:40:00,450 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:40:03,810 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:40:05,459 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:40:08,816 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:40:09,682 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 03:40:09,682 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 03:40:11,025 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 03:40:11,026 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 03:40:11,027 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 03:40:11,028 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 03:40:11,187 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 03:40:13,189 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 03:40:14,029 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:40:14,976 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:40:19,029 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:40:19,977 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:40:19,977 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:40:20,598 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 03:40:24,601 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:40:24,682 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 03:40:24,682 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 03:40:29,818 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:40:29,979 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:40:29,979 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:40:34,818 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:40:34,979 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:40:34,980 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:40:39,689 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 03:40:39,690 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 03:40:40,797 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:40:43,235 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 03:40:43,237 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 03:40:43,237 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 03:40:43,238 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 03:40:44,220 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 03:40:44,986 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:40:45,221 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 03:40:46,239 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:40:49,987 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:40:50,602 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 03:40:51,604 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:40:54,903 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 03:40:54,904 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 03:40:57,057 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:40:59,988 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:41:02,058 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:41:04,989 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:41:07,067 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:41:09,813 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 03:41:09,814 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 03:41:12,505 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 03:41:12,507 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 03:41:12,507 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 03:41:12,508 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:41:12,509 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 03:41:13,249 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 03:41:15,000 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:41:15,250 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 03:41:17,510 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:41:20,000 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:41:20,606 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 03:41:22,608 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:41:24,813 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 03:41:24,814 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 03:41:27,977 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:41:30,002 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:41:32,978 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:41:35,003 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:41:37,980 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:41:39,822 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 03:41:39,824 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 03:41:41,082 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 03:41:41,083 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 03:41:41,084 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 03:41:41,085 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 03:41:41,274 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 03:41:43,086 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:41:43,276 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 03:41:45,287 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:41:48,086 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:41:50,288 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:41:50,288 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:41:50,610 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 03:41:53,612 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:41:54,817 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 03:41:54,818 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 03:41:58,933 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:42:00,289 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:42:03,934 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:42:05,290 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:42:08,936 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:42:09,830 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 03:42:09,833 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 03:42:14,013 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:42:14,281 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 03:42:14,283 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 03:42:14,283 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 03:42:14,285 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 03:42:14,304 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 03:42:15,305 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 03:42:15,427 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:42:19,286 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:42:20,427 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:42:20,428 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:42:20,615 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 03:42:24,617 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:42:24,822 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 03:42:24,822 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 03:42:29,983 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:42:30,514 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:42:30,515 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:42:34,984 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:42:35,515 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:42:35,515 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:42:39,824 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 03:42:39,824 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 03:42:40,940 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:42:44,417 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 03:42:44,419 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 03:42:44,419 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 03:42:44,421 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 03:42:45,329 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 03:42:45,330 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 03:42:45,822 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:42:45,823 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:42:46,422 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:42:50,617 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 03:42:50,823 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:42:51,618 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:42:54,824 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 03:42:54,825 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 03:42:56,985 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:42:59,825 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:43:01,986 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:43:04,826 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:43:06,987 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:43:09,824 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 03:43:09,825 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 03:43:12,985 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:43:14,831 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:43:16,664 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 03:43:16,666 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 03:43:16,666 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 03:43:16,668 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 03:43:17,357 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 03:43:18,669 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:43:19,359 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 03:43:20,071 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:43:20,621 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 03:43:24,623 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:43:24,824 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 03:43:24,825 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 03:43:29,956 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:43:30,139 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:43:30,139 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:43:34,957 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:43:35,140 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:43:35,140 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:43:39,831 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 03:43:39,832 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 03:43:39,962 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:43:44,421 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 03:43:44,422 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 03:43:44,423 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 03:43:44,424 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 03:43:45,149 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:43:45,383 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 03:43:45,383 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 03:43:45,425 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:43:50,147 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:43:50,425 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:43:50,624 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 03:43:54,828 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 03:43:54,829 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 03:43:55,988 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:44:00,149 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:44:00,988 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:44:05,149 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:44:05,989 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:44:09,838 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 03:44:09,845 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 03:44:10,989 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:44:15,153 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:44:15,566 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 03:44:15,568 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 03:44:15,568 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 03:44:15,570 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 03:44:16,413 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 03:44:16,570 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:44:17,414 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 03:44:20,154 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:44:20,629 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 03:44:21,631 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:44:24,831 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 03:44:24,832 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 03:44:26,988 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:44:30,156 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:44:31,989 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:44:35,157 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:44:36,989 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:44:39,843 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 03:44:39,846 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 03:44:42,015 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:44:45,164 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:44:45,447 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 03:44:45,449 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 03:44:45,449 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 03:44:45,450 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 03:44:46,442 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 03:44:47,443 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 03:44:47,451 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:44:50,165 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:44:50,633 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 03:44:52,635 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:44:54,837 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 03:44:54,838 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 03:44:58,004 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:45:00,167 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:45:03,004 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:45:05,168 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:45:08,005 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:45:09,837 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 03:45:09,838 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 03:45:13,011 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:45:15,183 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:45:15,887 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 03:45:15,888 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 03:45:15,889 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 03:45:15,890 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 03:45:16,469 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 03:45:17,471 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 03:45:18,891 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:45:20,299 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:45:20,300 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:45:20,638 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 03:45:24,640 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:45:24,837 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 03:45:24,838 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 03:45:30,005 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:45:30,301 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:45:30,301 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:45:35,006 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:45:35,302 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:45:35,302 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:45:39,837 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 03:45:39,838 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 03:45:40,965 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:45:45,312 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:45:45,978 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:45:47,047 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 03:45:47,049 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 03:45:47,049 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 03:45:47,050 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 03:45:47,500 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 03:45:47,501 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 03:45:50,443 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:45:50,443 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:45:50,641 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 03:45:51,643 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:45:54,837 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 03:45:54,838 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 03:45:56,909 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:46:00,444 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:46:00,445 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:46:01,909 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:46:05,445 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:46:06,910 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:46:09,838 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 03:46:09,838 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 03:46:11,971 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:46:15,456 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:46:16,976 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:46:17,793 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 03:46:17,794 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 03:46:17,795 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 03:46:17,796 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 03:46:18,525 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 03:46:19,526 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 03:46:20,451 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:46:20,644 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 03:46:22,647 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:46:24,838 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 03:46:24,838 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 03:46:28,000 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:46:30,452 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:46:33,001 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:46:35,453 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:46:38,002 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:46:39,838 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 03:46:39,838 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 03:46:43,010 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:46:45,476 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:46:47,673 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 03:46:47,675 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 03:46:47,675 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 03:46:47,676 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 03:46:48,558 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 03:46:48,677 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:46:49,560 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 03:46:50,470 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:46:50,648 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 03:46:54,651 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:46:54,838 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 03:46:54,838 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 03:47:00,006 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:47:00,498 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:47:00,499 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:47:05,007 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:47:05,499 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:47:05,500 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:47:09,838 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 03:47:09,839 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 03:47:10,967 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:47:15,506 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:47:15,976 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:47:18,729 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 03:47:18,730 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 03:47:18,730 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 03:47:18,731 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 03:47:19,597 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 03:47:19,598 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 03:47:20,506 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:47:20,652 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 03:47:21,655 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:47:24,838 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 03:47:24,839 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 03:47:26,992 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:47:30,508 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:47:31,993 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:47:35,509 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:47:36,994 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:47:39,838 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 03:47:39,839 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 03:47:42,005 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:47:45,520 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:47:47,007 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:47:47,776 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 03:47:47,778 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 03:47:47,778 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 03:47:47,779 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 03:47:48,622 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 03:47:49,623 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 03:47:50,516 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:47:50,657 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 03:47:52,659 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:47:54,838 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 03:47:54,839 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 03:47:57,978 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:48:00,654 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:48:00,739 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:48:02,978 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:48:05,739 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:48:07,979 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:48:09,839 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 03:48:09,839 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 03:48:13,924 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:48:15,750 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:48:17,890 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 03:48:17,892 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 03:48:17,892 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 03:48:17,893 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 03:48:18,648 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 03:48:19,650 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 03:48:19,894 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:48:20,660 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 03:48:20,749 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:48:24,839 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 03:48:24,839 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 03:48:24,974 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:48:29,975 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:48:30,765 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:48:30,849 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:48:34,975 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:48:35,849 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:48:39,839 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 03:48:39,839 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 03:48:40,953 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:48:44,859 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:48:45,958 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:48:48,677 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 03:48:48,678 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 03:48:48,679 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 03:48:48,680 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 03:48:49,674 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 03:48:49,675 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 03:48:50,076 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:48:50,076 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:48:50,664 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 03:48:51,667 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:48:54,839 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 03:48:54,840 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 03:48:56,999 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:49:00,077 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:49:00,078 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:49:02,000 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:49:05,078 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:49:05,079 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:49:07,000 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:49:09,839 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 03:49:09,840 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 03:49:12,954 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:49:15,084 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:49:17,962 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:49:19,928 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 03:49:19,930 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 03:49:19,930 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 03:49:19,932 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 03:49:20,330 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:49:20,330 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:49:20,668 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 03:49:20,700 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 03:49:22,702 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 03:49:23,671 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:49:24,839 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 03:49:24,840 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 03:49:28,943 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:49:30,331 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:49:33,943 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:49:35,332 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:49:38,944 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:49:39,839 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 03:49:39,840 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 03:49:43,986 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:49:45,333 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:49:48,993 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:49:50,340 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:49:50,680 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 03:49:51,613 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 03:49:51,614 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 03:49:51,615 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 03:49:51,616 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 03:49:51,731 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 03:49:52,732 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 03:49:54,617 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:49:54,839 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 03:49:54,840 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 03:49:59,925 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:50:00,341 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:50:04,926 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:50:05,342 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:50:09,840 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 03:50:09,840 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 03:50:10,913 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:50:15,351 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:50:15,922 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:50:20,348 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:50:20,348 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:50:20,547 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 03:50:20,548 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 03:50:20,549 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 03:50:20,550 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 03:50:20,682 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 03:50:20,755 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 03:50:21,684 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:50:22,757 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 03:50:24,840 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 03:50:24,840 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 03:50:27,007 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:50:30,350 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:50:30,350 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:50:32,008 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:50:35,351 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:50:35,351 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:50:37,008 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:50:39,840 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 03:50:39,840 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 03:50:42,955 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:50:45,365 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:50:47,885 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 03:50:47,886 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 03:50:47,886 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 03:50:47,887 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 03:50:48,783 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 03:50:48,888 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:50:50,358 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:50:50,685 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 03:50:50,784 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 03:50:54,688 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:50:54,840 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 03:50:54,840 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 03:50:59,993 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:51:00,359 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:51:04,993 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:51:05,360 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:51:09,840 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 03:51:09,840 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 03:51:10,967 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:51:15,369 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:51:15,978 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:51:17,153 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 03:51:17,154 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 03:51:17,155 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 03:51:17,156 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 03:51:17,808 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 03:51:18,810 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 03:51:20,540 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:51:20,540 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:51:20,689 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 03:51:21,691 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:51:24,840 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 03:51:24,841 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 03:51:26,949 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:51:30,571 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:51:30,571 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:51:31,950 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:51:35,572 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:51:35,572 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:51:36,950 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:51:39,840 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 03:51:39,841 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 03:51:42,948 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:51:45,580 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:51:47,613 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 03:51:47,615 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 03:51:47,615 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 03:51:47,616 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 03:51:47,842 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 03:51:48,617 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:51:48,844 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 03:51:50,579 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:51:50,693 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 03:51:53,696 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:51:54,840 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 03:51:54,841 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 03:51:58,996 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:52:00,581 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:52:03,996 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:52:05,582 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:52:08,997 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:52:09,840 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 03:52:09,841 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 03:52:14,935 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:52:15,594 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:52:18,481 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 03:52:18,483 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 03:52:18,483 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 03:52:18,485 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 03:52:18,873 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 03:52:20,486 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:52:20,634 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:52:20,697 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 03:52:20,874 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 03:52:24,840 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 03:52:24,841 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 03:52:25,925 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:52:30,635 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:52:30,635 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:52:30,926 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:52:35,636 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:52:35,637 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:52:35,927 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:52:39,841 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 03:52:39,841 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 03:52:40,969 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:52:45,640 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:52:45,973 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:52:50,644 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:52:50,700 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 03:52:50,750 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 03:52:50,752 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 03:52:50,753 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 03:52:50,755 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 03:52:50,907 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 03:52:51,756 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:52:52,908 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 03:52:54,841 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 03:52:54,841 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 03:52:56,997 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:53:00,646 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:53:01,998 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:53:05,748 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:53:06,999 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:53:09,841 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 03:53:09,841 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 03:53:12,002 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:53:15,760 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:53:17,008 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:53:20,198 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 03:53:20,200 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 03:53:20,201 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 03:53:20,202 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 03:53:20,705 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 03:53:20,754 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:53:20,931 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 03:53:22,707 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:53:22,932 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 03:53:24,841 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 03:53:24,842 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 03:53:27,991 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:53:30,755 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:53:32,992 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:53:35,806 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:53:35,806 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:53:37,992 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:53:39,841 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 03:53:39,842 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 03:53:42,994 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:53:45,813 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:53:48,002 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:53:50,365 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 03:53:50,366 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 03:53:50,367 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 03:53:50,369 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 03:53:50,708 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 03:53:50,812 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:53:50,812 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:53:50,960 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 03:53:52,961 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 03:53:53,710 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:53:54,865 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 03:53:54,865 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 03:53:58,992 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:54:00,814 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:54:03,992 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:54:05,815 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:54:05,815 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:54:08,993 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:54:09,863 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 03:54:09,864 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 03:54:14,825 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:54:14,964 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:54:14,974 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 03:54:15,050 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:54:19,826 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:54:20,712 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 03:54:22,555 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:54:22,814 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:54:25,715 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:54:30,715 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:54:30,927 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:54:31,263 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:54:32,044 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 03:54:32,044 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 03:54:36,194 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:54:39,768 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:54:39,938 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:54:41,003 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 03:54:41,003 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 03:54:42,192 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:54:47,193 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:54:48,415 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:54:48,837 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:54:50,716 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 03:54:52,718 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:54:56,834 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:54:57,169 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:54:57,719 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:54:57,751 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 03:54:57,752 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 03:55:02,919 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:55:05,522 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:55:07,919 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:55:10,960 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:55:10,960 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:55:11,168 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 03:55:11,833 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 03:55:11,834 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 03:55:12,942 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:55:13,169 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 03:55:16,961 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:55:16,961 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:55:17,943 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:55:20,720 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 03:55:21,962 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:55:21,962 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:55:23,723 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:55:26,834 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 03:55:26,834 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 03:55:28,939 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:55:31,964 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:55:31,964 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:55:33,939 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:55:36,965 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:55:36,965 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:55:38,940 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:55:41,840 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 03:55:41,840 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 03:55:44,005 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:55:46,967 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:55:46,967 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:55:49,006 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:55:50,725 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 03:55:51,968 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:55:51,968 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:55:54,728 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:55:56,834 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 03:55:56,835 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 03:55:59,993 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:56:01,969 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:56:01,970 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:56:04,994 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:56:09,994 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:56:14,995 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:56:19,996 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:56:20,728 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 03:56:23,832 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:56:23,836 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:56:23,836 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 03:56:23,837 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 03:56:25,912 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:56:28,837 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:56:28,838 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:56:30,913 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:56:33,839 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:56:33,839 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:56:35,914 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:56:38,832 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 03:56:38,833 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 03:56:40,916 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:56:43,840 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:56:45,917 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:56:48,841 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:56:50,732 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 03:56:51,734 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:56:53,832 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 03:56:53,833 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 03:56:56,965 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:56:58,842 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:57:01,965 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:57:03,843 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:57:06,966 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:57:08,833 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 03:57:08,833 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 03:57:12,932 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:57:13,853 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:57:13,853 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:57:17,933 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:57:18,854 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:57:18,854 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:57:20,736 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 03:57:23,739 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:57:23,833 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 03:57:23,833 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 03:57:28,856 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:57:28,856 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:57:28,940 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:57:33,857 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:57:33,857 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:57:33,941 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:57:38,833 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 03:57:38,834 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 03:57:38,950 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:57:43,860 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:57:43,860 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:57:43,951 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:57:48,861 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:57:48,861 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:57:48,951 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:57:50,740 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 03:57:53,833 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 03:57:53,834 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 03:57:53,972 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:57:58,863 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:57:58,863 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:57:58,972 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:58:03,864 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:58:03,864 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:58:03,973 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:58:08,833 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 03:58:08,834 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 03:58:08,986 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:58:13,866 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:58:13,866 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:58:13,987 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:58:18,867 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:58:18,867 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:58:18,988 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:58:20,744 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 03:58:24,747 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:58:29,747 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:58:34,748 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:58:39,749 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:58:44,749 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:58:49,750 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:58:50,774 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 03:58:54,791 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:58:58,018 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:58:58,023 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:58:58,023 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 03:58:58,023 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 03:59:00,092 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:59:00,605 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 03:59:03,024 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:59:03,024 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:59:04,607 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 03:59:05,748 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:59:08,025 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:59:08,025 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:59:10,748 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:59:13,023 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 03:59:13,023 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 03:59:16,174 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:59:18,027 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:59:18,027 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:59:20,752 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 03:59:21,755 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:59:23,028 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:59:23,028 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:59:26,764 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:59:28,030 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 03:59:28,034 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 03:59:32,063 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 03:59:32,065 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 03:59:32,066 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 03:59:32,066 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:59:32,072 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 03:59:32,631 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 03:59:32,632 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 03:59:33,033 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:59:37,073 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:59:38,038 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:59:42,074 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:59:43,025 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 03:59:43,026 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 03:59:47,178 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:59:48,040 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:59:50,756 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 03:59:52,759 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:59:53,041 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 03:59:57,759 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 03:59:58,025 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 03:59:58,026 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 04:00:03,052 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:00:03,147 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:00:05,569 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 04:00:05,571 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 04:00:05,571 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 04:00:05,573 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 04:00:05,657 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 04:00:06,658 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 04:00:08,049 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:00:08,574 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:00:13,025 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 04:00:13,026 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 04:00:14,161 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:00:18,052 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:00:19,162 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:00:20,760 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 04:00:23,053 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:00:24,763 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:00:28,026 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 04:00:28,026 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 04:00:30,193 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:00:33,059 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:00:34,089 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 04:00:34,091 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 04:00:34,091 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 04:00:34,092 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 04:00:34,681 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 04:00:34,682 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 04:00:36,093 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:00:38,064 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:00:41,094 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:00:43,026 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 04:00:43,026 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 04:00:46,121 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:00:48,066 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:00:48,066 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:00:50,764 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 04:00:51,767 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:00:53,067 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:00:53,067 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:00:56,767 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:00:58,026 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 04:00:58,026 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 04:01:02,172 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:01:03,076 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:01:05,211 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 04:01:05,213 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 04:01:05,213 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 04:01:05,214 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 04:01:05,707 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 04:01:07,215 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:01:07,708 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 04:01:08,076 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:01:12,216 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:01:13,026 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 04:01:13,027 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 04:01:18,077 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:01:18,121 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:01:20,769 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 04:01:23,078 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:01:23,771 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:01:28,028 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 04:01:28,031 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 04:01:29,158 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:01:33,087 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:01:33,843 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 04:01:33,845 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 04:01:33,845 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 04:01:33,847 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 04:01:34,735 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 04:01:34,848 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:01:35,736 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 04:01:38,246 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:01:38,246 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:01:39,848 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:01:43,028 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 04:01:43,029 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 04:01:45,167 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:01:48,446 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:01:48,446 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:01:50,168 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:01:50,772 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 04:01:53,446 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:01:53,447 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:01:55,777 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:01:58,037 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 04:01:58,043 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 04:02:00,907 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 04:02:00,908 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 04:02:00,908 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 04:02:00,908 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:02:00,910 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 04:02:01,757 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 04:02:03,456 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:02:03,759 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 04:02:05,911 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:02:08,457 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:02:10,911 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:02:13,031 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 04:02:13,032 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 04:02:16,200 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:02:18,513 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:02:18,513 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:02:20,776 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 04:02:21,778 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:02:23,514 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:02:23,514 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:02:26,782 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:02:28,033 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 04:02:28,034 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 04:02:31,693 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 04:02:31,695 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 04:02:31,696 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 04:02:31,697 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 04:02:31,784 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 04:02:32,698 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:02:33,520 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:02:33,785 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 04:02:37,699 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:02:38,521 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:02:42,700 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:02:43,033 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 04:02:43,034 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 04:02:48,184 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:02:48,523 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:02:48,523 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:02:50,780 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 04:02:53,524 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:02:53,525 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:02:53,783 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:02:58,038 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 04:02:58,039 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 04:02:59,169 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:03:01,766 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 04:03:01,768 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 04:03:01,768 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 04:03:01,770 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 04:03:01,809 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 04:03:03,534 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:03:03,694 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:03:03,810 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 04:03:04,771 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:03:08,695 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:03:08,695 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:03:09,772 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:03:13,037 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 04:03:13,037 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 04:03:15,197 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:03:18,750 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:03:18,750 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:03:20,197 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:03:20,784 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 04:03:23,751 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:03:23,751 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:03:25,786 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:03:28,042 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 04:03:28,043 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 04:03:31,181 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:03:32,290 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 04:03:32,292 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 04:03:32,292 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 04:03:32,294 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 04:03:32,842 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 04:03:33,759 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:03:33,843 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 04:03:36,295 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:03:38,759 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:03:41,296 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:03:43,039 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 04:03:43,039 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 04:03:47,135 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:03:48,761 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:03:50,787 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 04:03:52,789 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:03:53,762 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:03:57,789 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:03:58,050 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 04:03:58,051 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 04:04:03,172 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:04:03,567 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 04:04:03,568 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 04:04:03,568 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 04:04:03,570 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 04:04:03,872 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 04:04:03,970 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:04:03,970 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:04:05,874 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 04:04:08,571 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:04:08,972 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:04:08,972 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:04:13,044 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 04:04:13,045 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 04:04:14,158 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:04:18,973 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:04:19,159 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:04:20,792 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 04:04:23,974 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:04:24,795 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:04:28,053 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 04:04:28,054 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 04:04:30,235 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:04:33,226 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 04:04:33,228 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 04:04:33,228 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 04:04:33,230 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 04:04:33,898 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 04:04:33,983 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:04:35,899 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 04:04:36,230 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:04:38,984 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:04:41,231 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:04:43,049 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 04:04:43,049 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 04:04:47,149 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:04:48,985 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:04:50,795 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 04:04:52,797 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:04:53,986 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:04:57,797 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:04:58,049 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 04:04:58,049 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 04:05:03,232 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:05:04,000 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:05:04,595 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 04:05:04,596 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 04:05:04,596 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 04:05:04,598 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 04:05:04,922 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 04:05:05,923 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 04:05:08,599 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:05:09,004 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:05:09,004 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:05:13,049 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 04:05:13,050 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 04:05:14,173 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:05:18,106 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:05:18,107 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:05:19,173 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:05:20,799 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 04:05:23,107 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:05:23,107 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:05:24,802 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:05:28,049 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 04:05:28,050 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 04:05:30,219 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:05:33,118 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:05:34,842 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 04:05:34,844 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 04:05:34,845 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 04:05:34,846 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 04:05:34,948 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 04:05:35,847 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:05:35,949 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 04:05:38,255 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:05:38,256 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:05:40,847 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:05:43,049 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 04:05:43,050 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 04:05:46,192 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:05:48,257 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:05:48,257 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:05:50,803 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 04:05:51,805 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:05:53,258 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:05:53,258 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:05:56,806 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:05:58,050 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 04:05:58,050 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 04:06:02,149 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:06:03,266 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:06:05,326 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 04:06:05,328 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 04:06:05,328 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 04:06:05,330 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 04:06:05,974 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 04:06:07,331 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:06:07,976 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 04:06:08,266 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:06:12,332 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:06:13,050 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 04:06:13,050 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 04:06:18,157 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:06:18,268 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:06:20,807 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 04:06:23,269 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:06:23,809 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:06:28,050 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 04:06:28,051 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 04:06:29,132 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:06:33,289 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:06:34,142 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:06:35,588 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 04:06:35,589 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 04:06:35,590 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 04:06:35,591 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 04:06:35,998 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 04:06:38,001 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 04:06:38,280 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:06:39,593 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:06:43,050 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 04:06:43,051 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 04:06:45,128 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:06:48,281 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:06:50,129 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:06:50,812 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 04:06:53,282 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:06:55,815 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:06:58,050 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 04:06:58,051 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 04:07:01,172 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:07:03,290 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:07:06,180 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:07:07,720 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 04:07:07,722 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 04:07:07,722 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 04:07:07,724 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 04:07:08,031 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 04:07:08,290 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:07:10,033 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 04:07:11,725 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:07:13,050 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 04:07:13,051 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 04:07:17,147 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:07:18,291 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:07:20,816 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 04:07:22,819 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:07:23,292 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:07:27,819 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:07:28,050 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 04:07:28,051 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 04:07:33,153 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:07:33,293 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:07:38,158 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:07:38,300 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:07:39,203 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 04:07:39,205 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 04:07:39,205 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 04:07:39,207 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 04:07:40,059 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 04:07:42,060 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 04:07:43,050 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 04:07:43,051 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 04:07:44,117 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:07:48,351 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:07:48,351 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:07:49,117 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:07:50,819 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 04:07:53,352 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:07:53,352 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:07:54,822 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:07:58,051 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 04:07:58,051 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 04:08:00,152 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:08:03,354 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:08:03,354 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:08:05,152 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:08:08,367 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:08:10,158 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:08:11,830 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 04:08:11,832 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 04:08:11,833 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 04:08:11,834 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 04:08:12,088 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 04:08:13,052 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 04:08:13,053 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 04:08:14,090 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 04:08:15,188 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:08:18,851 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:08:18,851 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:08:20,188 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:08:20,824 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 04:08:23,852 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:08:23,852 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:08:25,826 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:08:28,052 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 04:08:28,053 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 04:08:31,202 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:08:33,885 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:08:36,203 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:08:38,897 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:08:41,212 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:08:42,943 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 04:08:42,944 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 04:08:42,944 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 04:08:42,945 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 04:08:43,120 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 04:08:43,353 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 04:08:43,354 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 04:08:44,121 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 04:08:46,487 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:08:48,898 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:08:48,898 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:08:50,828 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 04:08:51,830 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:08:53,899 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:08:56,831 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:08:58,089 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 04:08:58,089 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 04:09:02,196 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:09:04,025 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:09:04,025 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:09:07,197 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:09:09,026 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:09:12,206 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:09:13,097 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 04:09:13,102 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 04:09:14,306 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 04:09:14,307 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 04:09:14,308 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 04:09:14,310 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 04:09:15,149 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 04:09:16,151 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 04:09:17,311 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:09:19,032 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:09:20,832 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 04:09:22,834 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:09:24,033 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:09:27,835 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:09:28,092 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 04:09:28,092 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 04:09:33,206 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:09:34,034 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:09:38,207 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:09:39,035 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:09:43,095 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 04:09:43,100 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 04:09:43,267 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:09:45,939 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 04:09:45,942 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 04:09:45,942 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 04:09:45,944 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 04:09:46,180 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 04:09:48,182 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 04:09:48,945 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:09:49,085 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:09:49,086 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:09:50,836 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 04:09:54,086 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:09:54,087 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:09:54,839 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:09:58,095 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 04:09:58,095 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 04:10:00,258 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:10:04,088 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:10:04,088 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:10:05,259 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:10:09,089 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:10:09,089 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:10:10,268 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:10:13,103 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 04:10:13,104 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 04:10:16,051 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 04:10:16,052 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 04:10:16,052 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 04:10:16,053 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:10:16,054 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 04:10:16,206 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 04:10:18,198 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:10:18,198 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:10:18,208 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 04:10:20,840 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 04:10:21,842 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:10:23,199 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:10:23,200 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:10:26,842 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:10:28,098 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 04:10:28,099 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 04:10:32,249 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:10:33,201 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:10:33,201 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:10:37,250 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:10:38,202 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:10:38,202 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:10:42,257 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:10:43,105 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 04:10:43,106 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 04:10:47,620 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 04:10:47,621 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 04:10:47,622 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 04:10:47,622 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:10:47,624 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 04:10:48,206 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:10:48,237 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 04:10:50,239 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 04:10:50,844 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 04:10:52,846 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:10:53,207 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:10:57,847 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:10:58,102 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 04:10:58,102 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 04:11:03,209 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:11:03,222 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:11:08,209 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:11:08,223 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:11:13,102 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 04:11:13,103 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 04:11:13,259 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:11:18,217 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:11:18,265 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:11:19,426 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 04:11:19,427 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 04:11:19,428 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 04:11:19,429 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 04:11:20,274 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 04:11:20,848 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 04:11:22,275 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 04:11:23,218 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:11:23,850 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:11:28,102 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 04:11:28,103 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 04:11:29,238 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:11:33,220 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:11:34,239 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:11:38,220 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:11:39,240 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:11:43,102 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 04:11:43,103 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 04:11:45,235 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:11:48,231 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:11:50,243 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:11:50,852 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 04:11:50,991 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 04:11:50,993 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 04:11:50,993 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 04:11:50,995 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 04:11:51,305 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 04:11:52,307 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 04:11:53,396 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:11:55,997 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:11:58,102 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 04:11:58,103 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 04:12:01,213 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:12:03,398 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:12:03,398 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:12:06,214 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:12:08,398 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:12:08,399 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:12:11,215 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:12:13,103 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 04:12:13,103 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 04:12:16,225 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:12:18,411 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:12:18,412 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:12:20,366 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 04:12:20,368 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 04:12:20,369 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 04:12:20,370 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 04:12:20,857 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 04:12:21,330 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 04:12:21,859 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:12:22,331 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 04:12:23,776 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:12:23,777 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:12:26,862 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:12:28,103 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 04:12:28,103 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 04:12:32,198 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:12:33,854 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:12:33,854 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:12:37,199 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:12:38,855 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:12:38,855 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:12:42,199 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:12:43,103 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 04:12:43,104 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 04:12:47,205 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:12:48,862 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:12:50,318 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 04:12:50,320 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 04:12:50,321 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 04:12:50,323 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 04:12:50,364 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 04:12:50,860 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 04:12:52,366 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 04:12:52,863 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:12:53,861 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:12:57,863 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:12:58,103 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 04:12:58,103 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 04:13:03,208 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:13:03,863 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:13:08,208 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:13:08,863 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:13:13,103 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 04:13:13,103 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 04:13:13,240 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:13:18,244 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:13:18,876 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:13:20,865 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 04:13:20,902 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 04:13:20,904 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 04:13:20,904 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 04:13:20,905 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 04:13:21,390 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 04:13:22,392 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 04:13:23,871 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:13:23,906 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:13:28,103 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 04:13:28,104 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 04:13:29,238 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:13:33,872 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:13:34,238 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:13:38,873 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:13:39,239 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:13:43,103 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 04:13:43,104 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 04:13:44,278 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:13:48,883 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:13:49,286 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:13:50,869 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 04:13:51,402 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 04:13:51,403 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 04:13:51,404 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 04:13:51,405 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 04:13:51,418 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 04:13:52,419 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 04:13:53,881 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:13:54,406 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:13:58,103 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 04:13:58,104 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 04:14:00,272 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:14:03,882 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:14:03,883 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:14:05,273 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:14:08,883 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:14:08,884 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:14:10,273 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:14:13,103 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 04:14:13,104 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 04:14:16,211 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:14:18,890 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:14:20,870 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 04:14:21,871 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:14:22,499 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 04:14:22,501 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 04:14:22,501 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 04:14:22,506 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 04:14:23,444 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 04:14:23,898 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:14:23,898 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:14:24,446 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 04:14:27,507 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:14:28,103 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 04:14:28,104 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 04:14:33,272 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:14:33,899 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:14:38,273 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:14:39,048 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:14:39,048 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:14:43,103 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 04:14:43,104 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 04:14:44,255 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:14:49,049 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:14:49,256 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:14:50,875 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 04:14:54,054 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:14:54,877 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:14:55,637 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 04:14:55,638 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 04:14:55,638 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 04:14:55,643 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 04:14:56,481 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 04:14:56,482 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 04:14:58,104 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 04:14:58,104 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 04:15:00,273 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:15:04,055 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:15:05,274 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:15:09,056 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:15:10,275 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:15:13,104 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 04:15:13,104 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 04:15:16,251 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:15:19,057 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:15:20,877 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 04:15:21,878 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:15:24,050 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 04:15:24,052 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 04:15:24,053 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 04:15:24,054 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 04:15:24,197 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:15:24,483 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:15:24,513 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 04:15:26,515 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 04:15:27,056 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:15:28,104 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 04:15:28,105 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 04:15:32,235 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:15:33,484 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:15:37,236 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:15:38,549 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:15:38,549 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:15:42,236 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:15:43,104 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 04:15:43,104 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 04:15:47,251 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:15:48,562 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:15:48,563 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:15:50,879 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 04:15:52,691 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 04:15:52,693 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 04:15:52,693 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 04:15:52,694 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:15:52,695 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 04:15:53,539 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 04:15:53,558 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:15:53,558 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:15:54,540 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 04:15:57,696 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:15:58,104 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 04:15:58,105 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 04:16:03,216 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:16:03,559 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:16:03,560 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:16:08,217 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:16:08,561 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:16:08,561 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:16:13,104 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 04:16:13,105 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 04:16:13,261 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:16:18,272 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:16:18,562 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:16:20,886 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 04:16:23,526 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 04:16:23,526 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 04:16:23,527 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 04:16:23,527 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:16:23,528 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 04:16:23,572 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 04:16:23,672 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:16:23,922 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:16:24,574 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 04:16:28,104 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 04:16:28,105 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 04:16:29,215 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:16:33,924 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:16:33,924 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:16:34,216 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:16:38,970 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:16:38,970 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:16:39,217 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:16:43,104 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 04:16:43,105 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 04:16:44,273 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:16:48,981 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:16:49,274 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:16:50,887 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 04:16:53,980 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:16:54,225 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 04:16:54,226 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 04:16:54,226 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 04:16:54,228 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 04:16:54,613 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 04:16:55,229 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:16:56,614 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 04:16:58,105 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 04:16:58,105 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 04:17:01,222 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:17:03,982 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:17:06,223 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:17:08,983 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:17:11,223 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:17:13,105 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 04:17:13,105 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 04:17:16,250 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:17:18,990 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:17:20,895 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 04:17:21,898 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:17:23,292 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 04:17:23,294 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 04:17:23,294 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 04:17:23,295 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 04:17:23,643 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 04:17:23,993 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:17:24,644 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 04:17:27,296 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:17:28,105 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 04:17:28,105 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 04:17:33,254 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:17:33,994 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:17:38,255 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:17:38,995 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:17:43,105 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 04:17:43,106 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 04:17:44,186 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:17:49,004 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:17:49,193 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:17:50,900 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 04:17:52,786 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 04:17:52,788 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 04:17:52,788 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 04:17:52,789 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 04:17:53,678 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 04:17:54,190 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:17:54,679 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 04:17:54,790 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:17:58,105 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 04:17:58,106 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 04:18:00,250 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:18:03,191 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:18:03,192 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:18:05,250 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:18:08,193 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:18:08,193 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:18:10,251 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:18:13,105 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 04:18:13,106 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 04:18:16,246 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:18:18,194 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:18:20,913 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 04:18:21,914 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:18:23,201 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:18:26,221 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 04:18:26,222 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 04:18:26,223 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 04:18:26,224 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 04:18:26,706 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 04:18:26,707 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 04:18:27,224 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:18:28,105 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 04:18:28,106 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 04:18:33,199 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:18:33,206 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:18:33,206 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:18:38,200 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:18:38,207 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:18:38,207 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:18:43,105 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 04:18:43,106 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 04:18:43,201 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:18:48,201 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:18:48,208 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:18:48,208 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:18:50,907 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 04:18:53,212 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:18:53,911 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:18:57,851 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 04:18:57,852 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 04:18:57,852 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 04:18:57,855 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 04:18:58,260 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:18:58,261 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 04:18:58,262 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 04:18:58,740 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 04:18:58,740 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 04:18:59,438 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:19:04,261 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:19:04,262 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:19:04,439 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:19:09,262 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:19:09,439 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:19:13,260 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 04:19:13,261 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 04:19:15,382 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:19:18,263 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:19:18,264 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:19:20,383 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:19:20,911 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 04:19:23,264 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:19:25,915 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:19:28,265 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 04:19:28,266 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 04:19:29,708 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 04:19:29,710 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 04:19:29,710 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 04:19:29,716 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 04:19:29,766 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 04:19:30,767 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 04:19:31,717 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:19:33,269 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:19:36,718 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:19:38,270 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:19:41,718 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:19:43,264 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 04:19:43,265 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 04:19:47,397 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:19:48,271 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:19:50,915 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 04:19:52,917 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:19:53,272 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:19:57,923 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:19:58,270 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 04:19:58,270 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 04:20:02,239 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 04:20:02,241 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 04:20:02,242 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 04:20:02,247 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 04:20:02,795 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 04:20:02,795 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 04:20:03,248 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:20:03,395 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:20:03,395 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:20:08,248 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:20:08,396 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:20:08,396 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:20:13,249 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:20:13,265 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 04:20:13,266 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 04:20:18,397 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:20:18,398 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:20:18,414 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:20:20,920 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 04:20:23,398 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:20:23,922 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:20:28,271 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 04:20:28,275 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 04:20:29,348 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:20:31,788 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 04:20:31,789 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 04:20:31,790 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 04:20:31,791 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 04:20:31,819 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 04:20:32,821 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 04:20:33,403 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:20:34,792 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:20:38,404 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:20:39,793 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:20:43,266 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 04:20:43,267 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 04:20:45,447 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:20:48,424 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:20:50,448 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:20:50,923 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 04:20:53,425 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:20:53,425 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:20:55,925 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:20:58,276 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 04:20:58,277 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 04:21:01,381 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:21:03,206 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 04:21:03,207 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 04:21:03,208 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 04:21:03,212 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 04:21:03,607 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:21:03,848 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 04:21:04,849 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 04:21:07,213 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:21:08,608 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:21:12,213 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:21:13,270 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 04:21:13,271 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 04:21:17,344 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:21:18,609 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:21:18,610 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:21:20,927 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 04:21:22,929 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:21:23,610 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:21:27,937 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:21:28,276 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 04:21:28,277 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 04:21:33,428 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:21:33,547 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 04:21:33,548 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 04:21:33,549 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 04:21:33,551 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 04:21:33,692 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:21:33,881 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 04:21:33,952 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:21:34,882 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 04:21:38,552 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:21:38,952 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:21:43,273 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 04:21:43,274 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 04:21:44,434 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:21:48,954 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:21:49,434 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:21:50,931 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 04:21:53,955 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:21:53,955 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:21:54,933 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:21:58,277 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 04:21:58,277 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 04:22:00,428 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:22:03,652 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 04:22:03,654 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 04:22:03,654 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 04:22:03,658 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 04:22:03,907 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 04:22:04,061 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:22:04,909 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 04:22:05,659 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:22:09,062 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:22:09,062 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:22:10,659 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:22:13,277 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 04:22:13,278 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 04:22:16,427 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:22:19,149 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:22:19,149 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:22:20,935 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 04:22:21,937 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:22:24,149 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:22:26,938 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:22:28,277 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 04:22:28,278 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 04:22:32,408 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:22:34,156 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:22:37,412 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:22:37,489 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 04:22:37,491 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 04:22:37,492 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 04:22:37,495 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 04:22:37,934 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 04:22:38,935 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 04:22:39,163 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:22:42,497 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:22:43,277 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 04:22:43,278 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 04:22:48,360 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:22:49,165 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:22:50,939 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 04:22:53,942 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:22:54,165 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:22:58,277 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 04:22:58,278 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 04:22:59,446 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:23:04,180 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:23:04,181 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:23:04,453 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:23:06,349 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 04:23:06,351 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 04:23:06,352 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 04:23:06,354 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 04:23:06,965 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 04:23:06,965 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 04:23:09,178 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:23:09,179 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:23:10,355 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:23:13,278 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 04:23:13,278 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 04:23:15,385 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:23:19,281 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:23:19,373 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:23:20,385 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:23:20,944 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 04:23:24,374 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:23:24,374 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:23:25,947 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:23:28,278 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 04:23:28,278 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 04:23:31,392 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:23:33,386 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:23:34,924 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 04:23:34,925 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 04:23:34,926 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 04:23:34,927 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 04:23:34,990 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 04:23:36,928 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:23:37,995 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 04:23:38,384 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:23:41,928 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:23:43,278 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 04:23:43,278 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 04:23:47,374 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:23:48,385 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:23:50,947 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 04:23:52,949 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:23:53,386 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:23:57,950 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:23:58,278 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 04:23:58,278 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 04:24:03,393 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:24:03,413 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:24:05,734 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 04:24:05,735 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 04:24:05,736 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 04:24:05,737 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 04:24:06,022 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 04:24:08,024 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 04:24:08,394 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:24:08,738 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:24:13,278 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 04:24:13,279 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 04:24:14,356 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:24:18,396 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:24:19,357 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:24:20,951 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 04:24:23,397 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:24:24,953 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:24:28,278 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 04:24:28,279 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 04:24:30,412 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:24:33,407 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:24:35,096 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 04:24:35,096 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 04:24:35,097 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 04:24:35,098 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 04:24:36,043 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 04:24:36,099 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:24:38,046 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 04:24:38,494 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:24:41,099 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:24:43,278 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 04:24:43,279 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 04:24:46,401 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:24:48,496 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:24:48,496 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:24:50,955 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 04:24:51,957 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:24:53,497 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:24:53,497 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:24:56,958 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:24:58,278 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 04:24:58,279 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 04:25:02,429 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:25:03,510 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:25:04,693 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 04:25:04,695 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 04:25:04,695 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 04:25:04,697 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 04:25:05,071 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 04:25:06,072 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 04:25:07,698 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:25:08,506 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:25:12,699 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:25:13,278 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 04:25:13,279 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 04:25:18,351 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:25:18,508 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:25:20,959 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 04:25:23,509 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:25:23,961 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:25:28,283 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 04:25:28,284 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 04:25:29,410 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:25:33,516 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:25:33,608 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 04:25:33,610 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 04:25:33,610 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 04:25:33,612 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 04:25:34,096 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 04:25:34,613 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:25:36,098 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 04:25:38,517 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:25:39,613 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:25:43,280 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 04:25:43,281 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 04:25:45,438 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:25:48,519 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:25:50,439 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:25:50,963 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 04:25:53,520 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:25:55,965 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:25:58,280 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 04:25:58,281 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 04:26:01,443 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:26:03,534 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:26:03,534 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:26:05,313 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 04:26:05,314 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 04:26:05,314 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 04:26:05,316 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 04:26:06,124 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 04:26:07,317 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:26:08,126 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 04:26:08,728 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:26:08,728 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:26:12,317 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:26:13,280 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 04:26:13,281 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 04:26:17,418 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:26:18,730 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:26:18,730 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:26:20,966 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 04:26:22,968 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:26:23,730 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:26:23,731 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:26:27,969 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:26:28,280 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 04:26:28,281 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 04:26:33,388 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:26:33,746 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:26:33,746 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:26:35,855 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 04:26:35,856 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 04:26:35,857 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 04:26:35,858 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 04:26:36,150 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 04:26:38,152 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 04:26:38,740 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:26:38,740 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:26:38,859 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:26:43,281 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 04:26:43,281 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 04:26:44,408 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:26:48,787 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:26:48,787 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:26:49,409 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:26:50,970 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 04:26:53,787 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:26:54,972 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:26:58,281 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 04:26:58,281 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 04:27:00,402 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:27:03,794 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:27:05,408 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:27:06,351 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 04:27:06,352 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 04:27:06,353 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 04:27:06,354 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 04:27:07,178 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 04:27:08,183 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 04:27:08,798 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:27:11,355 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:27:13,281 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 04:27:13,282 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 04:27:16,361 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:27:18,800 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:27:18,801 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:27:20,974 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 04:27:21,976 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:27:23,801 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:27:26,976 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:27:28,281 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 04:27:28,281 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 04:27:32,443 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:27:33,808 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:27:36,729 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 04:27:36,730 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 04:27:36,731 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 04:27:36,731 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 04:27:37,212 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 04:27:37,732 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:27:38,213 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 04:27:39,121 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:27:39,122 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:27:42,733 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:27:43,281 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 04:27:43,281 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 04:27:48,455 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:27:49,123 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:27:49,123 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:27:50,978 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 04:27:53,980 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:27:54,124 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:27:54,125 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:27:58,281 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 04:27:58,282 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 04:27:59,397 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:28:04,133 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:28:04,405 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:28:05,930 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 04:28:05,932 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 04:28:05,932 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 04:28:05,934 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 04:28:06,243 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 04:28:08,244 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 04:28:09,344 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:28:09,935 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:28:13,281 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 04:28:13,282 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 04:28:15,438 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:28:18,345 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:28:18,345 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:28:20,438 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:28:20,982 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 04:28:23,346 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:28:23,346 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:28:25,984 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:28:28,281 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 04:28:28,282 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 04:28:31,397 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:28:33,360 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:28:36,402 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:28:37,127 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 04:28:37,129 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 04:28:37,130 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 04:28:37,131 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 04:28:37,271 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 04:28:38,272 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 04:28:38,535 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:28:42,133 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:28:43,281 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 04:28:43,282 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 04:28:47,436 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:28:48,536 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:28:48,537 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:28:50,986 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 04:28:52,988 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:28:53,538 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:28:53,538 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:28:57,989 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:28:58,282 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 04:28:58,282 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 04:29:03,432 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:29:03,539 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:29:03,540 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:29:08,441 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:29:08,547 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:29:10,995 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 04:29:10,996 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 04:29:10,997 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 04:29:10,998 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 04:29:11,306 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 04:29:12,307 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 04:29:13,282 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 04:29:13,282 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 04:29:14,392 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:29:18,941 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:29:18,941 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:29:19,393 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:29:20,990 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 04:29:23,941 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:29:23,942 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:29:24,992 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:29:28,282 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 04:29:28,282 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 04:29:30,436 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:29:33,943 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:29:33,943 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:29:35,437 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:29:38,952 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:29:40,444 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:29:43,282 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 04:29:43,282 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 04:29:43,357 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 04:29:43,423 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 04:29:43,424 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 04:29:43,426 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 04:29:44,328 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 04:29:46,330 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 04:29:46,427 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:29:48,951 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:29:50,994 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 04:29:51,996 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:29:53,951 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:29:56,997 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:29:58,282 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 04:29:58,282 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 04:30:02,427 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:30:03,953 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:30:07,431 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:30:08,962 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:30:11,702 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 04:30:11,703 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 04:30:11,703 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 04:30:11,704 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 04:30:12,355 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 04:30:12,705 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:30:13,282 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 04:30:13,283 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 04:30:14,357 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 04:30:18,439 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:30:18,586 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:30:20,998 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 04:30:23,586 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:30:24,000 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:30:28,282 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 04:30:28,283 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 04:30:29,440 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:30:33,587 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:30:34,441 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:30:38,594 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:30:39,446 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:30:43,282 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 04:30:43,283 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 04:30:43,434 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 04:30:43,436 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 04:30:43,437 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 04:30:43,438 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 04:30:44,389 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 04:30:44,389 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 04:30:45,439 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:30:48,837 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:30:50,440 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:30:51,002 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 04:30:53,838 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:30:53,838 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:30:56,004 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:30:58,282 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 04:30:58,283 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 04:31:01,408 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:31:03,840 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:31:03,840 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:31:06,408 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:31:08,841 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:31:08,841 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:31:11,419 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:31:13,290 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 04:31:13,291 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 04:31:15,341 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 04:31:15,342 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 04:31:15,343 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 04:31:15,344 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 04:31:15,412 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 04:31:16,414 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 04:31:17,345 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:31:18,847 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:31:21,006 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 04:31:23,009 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:31:23,848 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:31:28,010 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:31:28,287 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 04:31:28,288 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 04:31:33,429 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:31:33,849 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:31:38,430 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:31:38,850 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:31:43,294 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 04:31:43,295 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 04:31:44,415 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:31:44,955 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 04:31:44,956 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 04:31:44,956 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 04:31:44,957 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 04:31:45,445 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 04:31:46,446 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 04:31:48,853 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:31:49,959 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:31:51,010 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 04:31:53,854 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:31:55,012 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:31:58,291 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 04:31:58,292 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 04:32:00,490 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:32:03,855 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:32:05,491 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:32:08,856 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:32:10,495 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:32:13,296 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 04:32:13,299 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 04:32:15,096 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 04:32:15,098 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 04:32:15,099 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 04:32:15,100 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 04:32:15,468 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 04:32:16,101 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:32:16,469 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 04:32:18,867 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:32:21,014 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 04:32:22,016 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:32:23,868 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:32:27,017 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:32:28,295 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 04:32:28,296 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 04:32:32,424 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:32:33,869 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:32:37,425 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:32:38,870 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:32:42,434 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:32:43,304 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 04:32:43,305 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 04:32:47,052 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 04:32:47,054 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 04:32:47,054 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 04:32:47,056 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 04:32:47,494 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 04:32:48,057 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:32:48,496 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 04:32:48,885 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:32:51,018 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 04:32:53,886 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:32:53,887 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:32:54,020 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:32:58,300 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 04:32:58,301 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 04:32:59,433 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:33:03,888 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:33:03,888 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:33:04,433 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:33:08,888 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:33:09,434 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:33:13,306 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 04:33:13,307 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 04:33:14,468 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:33:16,408 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 04:33:16,410 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 04:33:16,411 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 04:33:16,412 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 04:33:16,521 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 04:33:18,523 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 04:33:18,894 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:33:20,413 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:33:21,021 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 04:33:23,894 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:33:26,024 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:33:28,304 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 04:33:28,304 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 04:33:31,381 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:33:33,896 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:33:36,381 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:33:38,896 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:33:41,396 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:33:43,313 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 04:33:43,314 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 04:33:46,219 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 04:33:46,220 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 04:33:46,220 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 04:33:46,221 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 04:33:46,547 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 04:33:47,222 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:33:48,549 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 04:33:48,905 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:33:51,026 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 04:33:53,028 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:33:53,906 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:33:58,029 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:33:58,306 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 04:33:58,307 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 04:34:03,444 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:34:03,990 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:34:03,991 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:34:08,444 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:34:08,991 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:34:08,992 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:34:13,309 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 04:34:13,310 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 04:34:14,404 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:34:15,818 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 04:34:15,819 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 04:34:15,819 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 04:34:15,820 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 04:34:16,595 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 04:34:18,597 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 04:34:19,223 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:34:19,224 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:34:19,821 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:34:21,029 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 04:34:24,224 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:34:25,032 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:34:28,306 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 04:34:28,307 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 04:34:30,440 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:34:34,225 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:34:35,441 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:34:39,226 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:34:40,441 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:34:43,306 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 04:34:43,307 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 04:34:46,397 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:34:46,409 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 04:34:46,411 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 04:34:46,411 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 04:34:46,412 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 04:34:46,627 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 04:34:48,628 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 04:34:49,234 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:34:51,034 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 04:34:52,036 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:34:54,235 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:34:54,235 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:34:57,036 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:34:58,306 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 04:34:58,307 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 04:35:02,387 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:35:04,300 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:35:04,300 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:35:07,388 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:35:09,301 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:35:09,301 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:35:12,393 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:35:13,310 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 04:35:13,311 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 04:35:16,984 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 04:35:16,985 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 04:35:16,986 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 04:35:16,988 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 04:35:17,661 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 04:35:17,989 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:35:18,421 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:35:18,422 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:35:18,663 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 04:35:21,038 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 04:35:23,040 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:35:23,422 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:35:28,041 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:35:28,307 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 04:35:28,307 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 04:35:33,454 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:35:33,486 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:35:33,486 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:35:38,455 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:35:38,487 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:35:43,313 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 04:35:43,313 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 04:35:44,424 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:35:47,890 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 04:35:47,891 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 04:35:47,892 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 04:35:47,893 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 04:35:48,493 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:35:48,696 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 04:35:48,697 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 04:35:49,894 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:35:51,041 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 04:35:53,494 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:35:55,044 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:35:58,313 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 04:35:58,314 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 04:36:00,392 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:36:03,706 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:36:03,707 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:36:05,393 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:36:08,707 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:36:10,393 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:36:13,313 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 04:36:13,314 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 04:36:15,399 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:36:18,715 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:36:19,170 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 04:36:19,171 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 04:36:19,171 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 04:36:19,172 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 04:36:19,723 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 04:36:20,724 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 04:36:21,046 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 04:36:21,048 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:36:23,714 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:36:26,048 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:36:28,313 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 04:36:28,314 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 04:36:31,461 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:36:33,715 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:36:36,462 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:36:38,716 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:36:41,462 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:36:43,314 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 04:36:43,314 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 04:36:47,439 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:36:48,726 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:36:49,154 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 04:36:49,157 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 04:36:49,157 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 04:36:49,158 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 04:36:49,751 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 04:36:50,752 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 04:36:51,048 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 04:36:53,050 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:36:53,724 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:36:58,051 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:36:58,313 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 04:36:58,314 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 04:37:03,398 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:37:03,839 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:37:03,839 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:37:08,399 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:37:08,840 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:37:08,840 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:37:13,314 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 04:37:13,314 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 04:37:13,410 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:37:18,419 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:37:18,848 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:37:19,187 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 04:37:19,188 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 04:37:19,189 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 04:37:19,190 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 04:37:19,777 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 04:37:20,778 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 04:37:21,052 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 04:37:23,849 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:37:24,055 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:37:28,314 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 04:37:28,314 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 04:37:29,417 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:37:33,903 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:37:33,904 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:37:34,417 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:37:38,904 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:37:39,418 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:37:43,318 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 04:37:43,319 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 04:37:45,408 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:37:47,568 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 04:37:47,570 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 04:37:47,570 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 04:37:47,572 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 04:37:47,804 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 04:37:48,805 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 04:37:48,975 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:37:50,573 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:37:51,056 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 04:37:53,976 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:37:53,976 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:37:56,059 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:37:58,315 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 04:37:58,316 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 04:38:01,437 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:38:03,977 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:38:06,437 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:38:08,978 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:38:08,978 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:38:11,438 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:38:13,315 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 04:38:13,316 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 04:38:16,459 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:38:18,989 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:38:19,214 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 04:38:19,216 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 04:38:19,217 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 04:38:19,218 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 04:38:19,830 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 04:38:20,831 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 04:38:21,060 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 04:38:22,063 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:38:23,990 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:38:23,990 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:38:27,063 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:38:28,315 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 04:38:28,316 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 04:38:32,428 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:38:33,992 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:38:33,992 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:38:37,428 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:38:38,993 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:38:38,993 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:38:42,436 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:38:43,320 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 04:38:43,321 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 04:38:47,058 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 04:38:47,060 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 04:38:47,061 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 04:38:47,063 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 04:38:47,858 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 04:38:48,064 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:38:48,859 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 04:38:48,998 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:38:51,063 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 04:38:53,065 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:38:53,998 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:38:58,066 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:38:58,317 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 04:38:58,317 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 04:39:03,467 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:39:04,000 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:39:08,467 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:39:09,001 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:39:13,317 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 04:39:13,318 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 04:39:14,451 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:39:19,008 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:39:19,458 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:39:20,372 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 04:39:20,373 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 04:39:20,374 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 04:39:20,375 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 04:39:20,885 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 04:39:21,068 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 04:39:22,886 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 04:39:24,008 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:39:25,071 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:39:28,317 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 04:39:28,318 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 04:39:30,489 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:39:34,225 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:39:34,226 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:39:35,490 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:39:39,226 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:39:39,226 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:39:40,491 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:39:43,317 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 04:39:43,318 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 04:39:46,449 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:39:49,242 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:39:51,075 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 04:39:52,019 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 04:39:52,020 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 04:39:52,021 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 04:39:52,021 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:39:52,023 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 04:39:52,915 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 04:39:52,915 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 04:39:54,428 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:39:54,428 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:39:57,024 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:39:58,317 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 04:39:58,318 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 04:40:02,462 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:40:03,429 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:40:03,429 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:40:07,463 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:40:08,430 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:40:08,430 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:40:12,464 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:40:13,318 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 04:40:13,318 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 04:40:18,396 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:40:18,437 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:40:21,076 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 04:40:23,438 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:40:23,764 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 04:40:23,766 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 04:40:23,766 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 04:40:23,767 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:40:23,768 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 04:40:23,944 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 04:40:24,945 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 04:40:28,318 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 04:40:28,319 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 04:40:29,431 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:40:33,440 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:40:34,431 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:40:38,440 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:40:39,432 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:40:43,318 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 04:40:43,319 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 04:40:45,391 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:40:48,442 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:40:48,442 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:40:50,404 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:40:51,080 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 04:40:53,456 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:40:53,456 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:40:56,087 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:40:56,106 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 04:40:56,108 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 04:40:56,108 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 04:40:56,109 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 04:40:56,978 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 04:40:56,979 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 04:40:58,318 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 04:40:58,319 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 04:41:01,415 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:41:03,507 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:41:06,416 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:41:08,508 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:41:11,417 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:41:13,318 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 04:41:13,319 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 04:41:17,401 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:41:18,509 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:41:21,084 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 04:41:23,092 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:41:23,514 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:41:27,609 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 04:41:27,611 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 04:41:27,612 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 04:41:27,613 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 04:41:28,005 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 04:41:28,318 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 04:41:28,319 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 04:41:28,389 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:41:29,007 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 04:41:33,390 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:41:33,519 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:41:38,390 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:41:38,520 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:41:43,318 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 04:41:43,319 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 04:41:43,458 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:41:48,459 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:41:48,650 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:41:48,651 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:41:51,088 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 04:41:53,661 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:41:54,097 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:41:57,080 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 04:41:57,081 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 04:41:57,082 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 04:41:57,083 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 04:41:58,034 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 04:41:58,318 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 04:41:58,319 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 04:41:59,036 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 04:41:59,440 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:42:03,657 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:42:04,441 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:42:08,658 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:42:09,442 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:42:13,318 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 04:42:13,319 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 04:42:15,413 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:42:18,659 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:42:20,413 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:42:21,091 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 04:42:23,671 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:42:26,097 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:42:27,397 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 04:42:27,399 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 04:42:27,400 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 04:42:27,401 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 04:42:28,061 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 04:42:28,324 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 04:42:28,325 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 04:42:29,063 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 04:42:31,471 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:42:33,800 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:42:36,471 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:42:38,801 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:42:38,801 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:42:41,472 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:42:43,323 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 04:42:43,324 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 04:42:47,398 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:42:48,802 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:42:51,095 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 04:42:53,104 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:42:53,816 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:42:58,099 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 04:42:58,101 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 04:42:58,101 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 04:42:58,102 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 04:42:58,501 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 04:42:58,501 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 04:42:58,600 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:42:59,089 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 04:42:59,089 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 04:43:03,600 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:43:03,809 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:43:08,601 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:43:08,810 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:43:13,500 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 04:43:13,500 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 04:43:13,625 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:43:18,625 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:43:18,860 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:43:21,099 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 04:43:23,867 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:43:24,109 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:43:28,501 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 04:43:28,503 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 04:43:29,354 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 04:43:29,355 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 04:43:29,355 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 04:43:29,356 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:43:29,356 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 04:43:30,122 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 04:43:31,123 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 04:43:33,870 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:43:34,358 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:43:38,871 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:43:39,358 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:43:43,501 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 04:43:43,502 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 04:43:44,625 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:43:48,872 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:43:49,626 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:43:51,102 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 04:43:53,978 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:43:53,978 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:43:55,115 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:43:58,511 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 04:43:58,512 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 04:44:00,585 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 04:44:00,587 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 04:44:00,588 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 04:44:00,588 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:44:00,589 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 04:44:01,153 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 04:44:01,153 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 04:44:03,987 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:44:05,590 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:44:08,988 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:44:10,590 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:44:13,503 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 04:44:13,504 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 04:44:15,608 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:44:18,989 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:44:20,609 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:44:21,106 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 04:44:23,990 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:44:26,111 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:44:28,517 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 04:44:28,518 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 04:44:29,832 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 04:44:29,833 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 04:44:29,833 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 04:44:29,834 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 04:44:30,185 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 04:44:31,186 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 04:44:31,835 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:44:34,248 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:44:36,835 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:44:39,249 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:44:39,249 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:44:41,836 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:44:43,507 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 04:44:43,508 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 04:44:47,659 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:44:49,251 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:44:49,251 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:44:51,111 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 04:44:53,113 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:44:54,252 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:44:58,121 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:44:58,520 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 04:44:58,523 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 04:45:00,750 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 04:45:00,752 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 04:45:00,752 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 04:45:00,753 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 04:45:01,215 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 04:45:03,217 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 04:45:03,754 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:45:04,262 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:45:08,755 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:45:09,262 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:45:13,513 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 04:45:13,514 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 04:45:14,616 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:45:19,349 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:45:19,349 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:45:19,616 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:45:21,114 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 04:45:24,360 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:45:25,118 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:45:28,518 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 04:45:28,519 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 04:45:29,952 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 04:45:29,954 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 04:45:29,954 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 04:45:29,956 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 04:45:30,241 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 04:45:30,957 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:45:31,242 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 04:45:34,357 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:45:35,957 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:45:39,358 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:45:40,958 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:45:43,515 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 04:45:43,516 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 04:45:46,598 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:45:49,440 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:45:51,119 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 04:45:52,122 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:45:54,445 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:45:57,127 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:45:57,221 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 04:45:57,222 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 04:45:57,222 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 04:45:57,223 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 04:45:57,259 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 04:45:58,515 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 04:45:58,516 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 04:45:59,261 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 04:46:02,609 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:46:03,626 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:46:03,626 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:46:07,609 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:46:08,627 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:46:08,627 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:46:12,610 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:46:13,515 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 04:46:13,516 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 04:46:17,666 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:46:18,728 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:46:18,728 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:46:21,122 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 04:46:23,127 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:46:23,741 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:46:28,131 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:46:28,516 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 04:46:28,516 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 04:46:28,722 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 04:46:28,724 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 04:46:28,724 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 04:46:28,726 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 04:46:29,288 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 04:46:30,289 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 04:46:33,727 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:46:34,127 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:46:34,127 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:46:38,728 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:46:39,127 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:46:39,128 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:46:43,515 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 04:46:43,516 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 04:46:44,648 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:46:49,148 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:46:49,149 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:46:49,649 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:46:51,127 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 04:46:54,149 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:46:54,150 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:46:55,129 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:46:58,523 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 04:46:58,527 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 04:47:00,639 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:47:01,343 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 04:47:01,345 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 04:47:01,345 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 04:47:01,347 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 04:47:02,316 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 04:47:04,161 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:47:04,161 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:47:04,318 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 04:47:06,348 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:47:09,162 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:47:11,348 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:47:13,518 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 04:47:13,519 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 04:47:16,673 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:47:19,321 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:47:19,321 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:47:21,131 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 04:47:22,134 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:47:24,321 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:47:24,322 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:47:27,144 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:47:28,525 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 04:47:28,526 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 04:47:32,607 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:47:32,672 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 04:47:32,673 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 04:47:32,674 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 04:47:32,675 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 04:47:33,347 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 04:47:34,327 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:47:34,348 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 04:47:37,677 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:47:39,328 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:47:42,677 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:47:43,522 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 04:47:43,523 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 04:47:47,689 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:47:49,330 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:47:51,135 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 04:47:53,137 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:47:54,330 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:47:58,138 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:47:58,522 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 04:47:58,523 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 04:48:03,623 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:48:04,343 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:48:05,325 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 04:48:05,326 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 04:48:05,326 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 04:48:05,327 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 04:48:05,377 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 04:48:06,379 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 04:48:09,328 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:48:09,477 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:48:09,477 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:48:13,522 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 04:48:13,523 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 04:48:14,679 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:48:19,478 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:48:19,680 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:48:21,137 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 04:48:24,479 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:48:25,139 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:48:28,522 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 04:48:28,523 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 04:48:30,590 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:48:34,493 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:48:35,595 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:48:36,149 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 04:48:36,151 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 04:48:36,151 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 04:48:36,153 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 04:48:36,409 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 04:48:38,411 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 04:48:39,561 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:48:39,561 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:48:41,154 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:48:43,522 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 04:48:43,523 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 04:48:46,657 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:48:48,562 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:48:51,141 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 04:48:52,144 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:48:53,563 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:48:53,563 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:48:57,145 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:48:58,523 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 04:48:58,523 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 04:49:02,627 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:49:03,570 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:49:07,636 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:49:07,696 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 04:49:07,698 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 04:49:07,698 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 04:49:07,700 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 04:49:08,437 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 04:49:08,572 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:49:10,438 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 04:49:12,701 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:49:13,523 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 04:49:13,523 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 04:49:18,574 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:49:18,672 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:49:21,145 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 04:49:23,575 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:49:24,148 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:49:28,523 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 04:49:28,523 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 04:49:29,660 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:49:33,576 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:49:34,660 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:49:38,580 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:49:39,668 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:49:40,828 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 04:49:40,829 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 04:49:40,829 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 04:49:40,831 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 04:49:41,464 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 04:49:42,466 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 04:49:43,523 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 04:49:43,524 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 04:49:45,651 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:49:48,786 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:49:48,786 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:49:50,652 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:49:51,149 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 04:49:53,786 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:49:53,787 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:49:56,152 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:49:58,523 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 04:49:58,524 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 04:50:01,678 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:50:03,788 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:50:03,788 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:50:06,683 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:50:08,796 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:50:11,685 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:50:12,104 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 04:50:12,106 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 04:50:12,106 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 04:50:12,107 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 04:50:12,496 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 04:50:13,523 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 04:50:13,524 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 04:50:14,497 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 04:50:17,667 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:50:18,932 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:50:21,154 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 04:50:23,157 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:50:23,932 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:50:23,933 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:50:28,157 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:50:28,523 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 04:50:28,524 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 04:50:33,592 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:50:33,934 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:50:38,597 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:50:38,947 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:50:41,959 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 04:50:41,961 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 04:50:41,962 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 04:50:41,963 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 04:50:42,565 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 04:50:43,524 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 04:50:43,524 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 04:50:43,659 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:50:44,567 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 04:50:48,660 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:50:49,035 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:50:49,035 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:50:51,158 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 04:50:54,036 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:50:54,036 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:50:54,161 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:50:58,524 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 04:50:58,524 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 04:50:59,693 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:51:04,037 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:51:04,038 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:51:04,694 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:51:09,047 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:51:09,703 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:51:13,042 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 04:51:13,043 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 04:51:13,044 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 04:51:13,045 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 04:51:13,524 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 04:51:13,525 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 04:51:13,589 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 04:51:14,591 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 04:51:15,675 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:51:19,047 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:51:20,676 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:51:21,161 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 04:51:24,048 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:51:26,164 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:51:28,640 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 04:51:28,640 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 04:51:31,761 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:51:34,049 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:51:36,761 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:51:39,050 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:51:41,765 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:51:43,552 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 04:51:43,556 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 04:51:46,346 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 04:51:46,348 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 04:51:46,349 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 04:51:46,350 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 04:51:46,624 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 04:51:47,351 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:51:48,626 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 04:51:49,062 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:51:51,165 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 04:51:53,168 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:51:54,062 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:51:58,168 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:51:58,545 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 04:51:58,546 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 04:52:03,708 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:52:04,064 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:52:08,708 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:52:09,070 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:52:13,549 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 04:52:13,549 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 04:52:14,646 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:52:14,767 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 04:52:14,769 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 04:52:14,770 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 04:52:14,771 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 04:52:15,651 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 04:52:16,652 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 04:52:19,172 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:52:19,773 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:52:21,169 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 04:52:24,173 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:52:25,172 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:52:28,547 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 04:52:28,548 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 04:52:30,647 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:52:34,175 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:52:35,648 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:52:39,176 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:52:40,649 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:52:43,559 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 04:52:43,560 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 04:52:45,559 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 04:52:45,560 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 04:52:45,561 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 04:52:45,562 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 04:52:45,679 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 04:52:46,563 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:52:46,680 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 04:52:49,186 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:52:51,174 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 04:52:52,176 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:52:54,186 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:52:57,176 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:52:58,552 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 04:52:58,553 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 04:53:02,623 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:53:04,188 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:53:07,623 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:53:09,188 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:53:12,627 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:53:13,554 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 04:53:13,556 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 04:53:17,639 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 04:53:17,640 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 04:53:17,641 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 04:53:17,641 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:53:17,642 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 04:53:17,708 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 04:53:18,709 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 04:53:19,197 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:53:21,177 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 04:53:23,179 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:53:24,197 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:53:28,180 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:53:28,554 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 04:53:28,555 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 04:53:33,664 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:53:34,199 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:53:38,665 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:53:39,200 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:53:43,562 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 04:53:43,566 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 04:53:44,661 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:53:47,086 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 04:53:47,087 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 04:53:47,087 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 04:53:47,088 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 04:53:47,738 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 04:53:48,738 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 04:53:49,232 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:53:50,089 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:53:51,181 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 04:53:54,232 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:53:55,183 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:53:58,558 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 04:53:58,558 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 04:54:00,703 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:54:04,249 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:54:05,704 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:54:09,249 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:54:10,704 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:54:13,569 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 04:54:13,570 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 04:54:15,707 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:54:18,084 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 04:54:18,086 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 04:54:18,087 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 04:54:18,088 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 04:54:18,768 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 04:54:19,497 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:54:19,498 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:54:20,770 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 04:54:21,090 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:54:21,185 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 04:54:24,498 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:54:24,498 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:54:26,187 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:54:28,565 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 04:54:28,565 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 04:54:31,701 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:54:34,500 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:54:34,500 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:54:36,701 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:54:39,501 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:54:39,501 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:54:41,702 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:54:43,570 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 04:54:43,571 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 04:54:47,684 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:54:48,352 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 04:54:48,353 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 04:54:48,354 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 04:54:48,355 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 04:54:48,750 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:54:48,750 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:54:48,805 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 04:54:50,807 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 04:54:51,189 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 04:54:53,191 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:54:53,751 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:54:58,192 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:54:58,568 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 04:54:58,568 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 04:55:03,640 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:55:03,752 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:55:03,752 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:55:08,640 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:55:08,753 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:55:08,753 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:55:13,575 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 04:55:13,578 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 04:55:13,661 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:55:18,507 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 04:55:18,509 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 04:55:18,510 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 04:55:18,511 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 04:55:18,840 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 04:55:18,904 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:55:19,512 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:55:20,842 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 04:55:21,194 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 04:55:23,906 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:55:23,906 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:55:25,196 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:55:28,572 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 04:55:28,572 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 04:55:30,642 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:55:33,978 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:55:33,978 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:55:35,642 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:55:38,979 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:55:40,642 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:55:43,574 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 04:55:43,578 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 04:55:45,654 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:55:47,519 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 04:55:47,520 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 04:55:47,521 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 04:55:47,522 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 04:55:47,867 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 04:55:48,870 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 04:55:48,989 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:55:48,990 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:55:51,197 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 04:55:51,203 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:55:53,990 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:55:53,990 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:55:56,200 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:55:58,574 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 04:55:58,574 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 04:56:01,641 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:56:04,211 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:56:04,211 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:56:06,641 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:56:09,219 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:56:11,650 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:56:13,580 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 04:56:13,581 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 04:56:15,015 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 04:56:15,016 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 04:56:15,016 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 04:56:15,018 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 04:56:15,893 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 04:56:16,894 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 04:56:17,019 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:56:19,419 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:56:21,201 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 04:56:22,203 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:56:24,420 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:56:27,204 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:56:28,578 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 04:56:28,578 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 04:56:32,739 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:56:34,421 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:56:37,739 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:56:39,428 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:56:42,743 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:56:43,586 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 04:56:43,587 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 04:56:44,981 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 04:56:44,982 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 04:56:44,982 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 04:56:44,984 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 04:56:45,926 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 04:56:46,928 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 04:56:47,985 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:56:49,431 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:56:49,431 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:56:51,205 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 04:56:53,207 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:56:54,431 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:56:54,432 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:56:58,208 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:56:58,581 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 04:56:58,582 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 04:57:03,655 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:57:04,433 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:57:04,433 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:57:08,656 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:57:09,441 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:57:13,596 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 04:57:13,597 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 04:57:13,757 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:57:14,941 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 04:57:14,943 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 04:57:14,943 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 04:57:14,944 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 04:57:14,957 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 04:57:16,959 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 04:57:18,945 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:57:19,443 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:57:21,209 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 04:57:24,212 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:57:24,444 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:57:28,587 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 04:57:28,588 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 04:57:29,735 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:57:34,582 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:57:34,735 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:57:39,586 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:57:39,586 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:57:39,742 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:57:43,597 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 04:57:43,598 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 04:57:44,329 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 04:57:44,331 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 04:57:44,331 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 04:57:44,332 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 04:57:44,989 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 04:57:45,332 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:57:46,992 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 04:57:48,728 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:57:48,728 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:57:50,333 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:57:51,213 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 04:57:53,729 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:57:53,729 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:57:56,215 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:57:58,591 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 04:57:58,591 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 04:58:01,719 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:58:03,731 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:58:03,731 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:58:06,719 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:58:08,732 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:58:08,732 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:58:11,722 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:58:13,596 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 04:58:13,597 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 04:58:15,078 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 04:58:15,079 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 04:58:15,080 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 04:58:15,081 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 04:58:16,020 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 04:58:17,022 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 04:58:17,083 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:58:18,739 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:58:18,739 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:58:21,218 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 04:58:22,220 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:58:23,740 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:58:23,740 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:58:27,221 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:58:28,591 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 04:58:28,592 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 04:58:32,706 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:58:33,741 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:58:33,741 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:58:37,707 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:58:38,742 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:58:38,742 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:58:42,720 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:58:43,593 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 04:58:43,593 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 04:58:48,215 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 04:58:48,216 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 04:58:48,217 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 04:58:48,217 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:58:48,220 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 04:58:48,748 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:58:49,049 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 04:58:49,050 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 04:58:51,221 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 04:58:53,223 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:58:53,749 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:58:58,224 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:58:58,591 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 04:58:58,592 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 04:59:03,670 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:59:03,837 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:59:08,671 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:59:08,838 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:59:13,598 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 04:59:13,599 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 04:59:13,746 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:59:18,343 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 04:59:18,345 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 04:59:18,345 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 04:59:18,346 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 04:59:18,845 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:59:19,074 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 04:59:19,075 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 04:59:19,347 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:59:21,224 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 04:59:23,846 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:59:25,226 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:59:28,598 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 04:59:28,599 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 04:59:30,677 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:59:33,848 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:59:35,678 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:59:38,849 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:59:40,678 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:59:43,602 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 04:59:43,604 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 04:59:45,701 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:59:48,355 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 04:59:48,357 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 04:59:48,357 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 04:59:48,358 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 04:59:48,859 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:59:49,108 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 04:59:49,109 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 04:59:51,228 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 04:59:51,234 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:59:53,860 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 04:59:56,230 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 04:59:58,601 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 04:59:58,602 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 05:00:01,680 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:00:03,861 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:00:06,681 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:00:08,862 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:00:11,682 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:00:13,601 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 05:00:13,602 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 05:00:16,711 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:00:18,872 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:00:20,275 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 05:00:20,276 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 05:00:20,277 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 05:00:20,278 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 05:00:21,143 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 05:00:21,144 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 05:00:21,232 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 05:00:22,234 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:00:23,873 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:00:23,874 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:00:27,235 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:00:28,602 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 05:00:28,602 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 05:00:32,699 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:00:33,875 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:00:33,875 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:00:37,699 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:00:38,876 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:00:38,876 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:00:42,700 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:00:43,601 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 05:00:43,602 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 05:00:47,750 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:00:48,888 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:00:50,009 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 05:00:50,011 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 05:00:50,011 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 05:00:50,012 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 05:00:50,167 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 05:00:51,168 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 05:00:51,237 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 05:00:53,239 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:00:53,887 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:00:58,240 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:00:58,601 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 05:00:58,602 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 05:01:03,750 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:01:03,888 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:01:03,889 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:01:08,750 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:01:08,889 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:01:08,890 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:01:13,602 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 05:01:13,602 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 05:01:13,768 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:01:18,779 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:01:18,898 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:01:20,820 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 05:01:20,821 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 05:01:20,821 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 05:01:20,823 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 05:01:21,196 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 05:01:21,240 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 05:01:23,198 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 05:01:24,222 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:01:24,222 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:01:24,242 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:01:28,602 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 05:01:28,602 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 05:01:29,674 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:01:34,224 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:01:34,674 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:01:39,224 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:01:39,225 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:01:39,675 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:01:43,602 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 05:01:43,602 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 05:01:44,749 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:01:49,238 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:01:49,754 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:01:50,939 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 05:01:50,941 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 05:01:50,941 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 05:01:50,943 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 05:01:51,216 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 05:01:51,244 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 05:01:53,218 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 05:01:54,347 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:01:55,246 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:01:58,602 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 05:01:58,603 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 05:02:00,752 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:02:04,348 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:02:05,753 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:02:09,348 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:02:10,753 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:02:13,602 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 05:02:13,603 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 05:02:16,720 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:02:19,354 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:02:20,597 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 05:02:20,598 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 05:02:20,598 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 05:02:20,600 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 05:02:21,248 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 05:02:21,249 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 05:02:21,249 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 05:02:22,251 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:02:24,354 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:02:27,252 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:02:28,603 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 05:02:28,603 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 05:02:32,764 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:02:34,355 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:02:37,765 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:02:39,356 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:02:42,765 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:02:43,607 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 05:02:43,616 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 05:02:48,657 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 05:02:48,659 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 05:02:48,660 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 05:02:48,660 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:02:48,661 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 05:02:49,276 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 05:02:49,276 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 05:02:49,364 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:02:51,252 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 05:02:54,254 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:02:54,364 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:02:54,365 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:02:58,606 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 05:02:58,606 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 05:02:59,747 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:03:04,366 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:03:04,366 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:03:04,748 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:03:09,367 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:03:09,367 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:03:09,748 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:03:13,614 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 05:03:13,618 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 05:03:15,730 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:03:18,705 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 05:03:18,707 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 05:03:18,707 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 05:03:18,708 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 05:03:19,298 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 05:03:19,298 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 05:03:19,376 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:03:21,256 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 05:03:21,262 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:03:24,377 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:03:26,258 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:03:28,611 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 05:03:28,612 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 05:03:31,775 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:03:34,378 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:03:36,776 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:03:39,379 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:03:41,776 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:03:43,611 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 05:03:43,611 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 05:03:47,749 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:03:48,402 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 05:03:48,403 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 05:03:48,404 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 05:03:48,406 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 05:03:49,331 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 05:03:49,331 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 05:03:49,387 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:03:49,388 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:03:51,260 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 05:03:53,262 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:03:54,389 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:03:54,389 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:03:58,263 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:03:58,611 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 05:03:58,612 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 05:04:03,724 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:04:04,391 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:04:04,391 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:04:08,725 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:04:09,392 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:04:09,392 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:04:13,615 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 05:04:13,618 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 05:04:14,707 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:04:17,427 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 05:04:17,428 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 05:04:17,428 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 05:04:17,430 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 05:04:18,351 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 05:04:19,352 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 05:04:19,406 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:04:19,407 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:04:20,431 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:04:21,263 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 05:04:24,407 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:04:26,266 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:04:28,614 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 05:04:28,615 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 05:04:31,766 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:04:34,408 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:04:36,766 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:04:39,409 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:04:41,776 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:04:43,624 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 05:04:43,624 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 05:04:46,610 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 05:04:46,611 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 05:04:46,612 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 05:04:46,613 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 05:04:47,380 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 05:04:47,380 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 05:04:47,613 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:04:49,416 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:04:49,416 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:04:51,267 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 05:04:53,270 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:04:54,417 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:04:54,417 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:04:58,271 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:04:58,618 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 05:04:58,619 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 05:05:03,713 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:05:04,418 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:05:04,419 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:05:08,713 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:05:09,419 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:05:09,420 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:05:13,627 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 05:05:13,630 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 05:05:14,708 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:05:15,986 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 05:05:15,987 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 05:05:15,988 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 05:05:15,989 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 05:05:16,409 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 05:05:17,410 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 05:05:19,424 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:05:19,990 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:05:21,271 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 05:05:24,424 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:05:25,274 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:05:28,621 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 05:05:28,621 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 05:05:30,718 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:05:34,426 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:05:35,719 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:05:39,426 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:05:40,719 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:05:43,630 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 05:05:43,634 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 05:05:45,727 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:05:48,585 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 05:05:48,587 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 05:05:48,587 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 05:05:48,589 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 05:05:49,435 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:05:49,444 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 05:05:49,444 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 05:05:51,276 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 05:05:51,278 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:05:54,436 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:05:56,278 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:05:58,624 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 05:05:58,624 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 05:06:01,776 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:06:04,437 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:06:06,776 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:06:09,438 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:06:11,777 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:06:13,623 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 05:06:13,623 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 05:06:17,691 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:06:18,726 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 05:06:18,728 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 05:06:18,728 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 05:06:18,730 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 05:06:19,445 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:06:19,464 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 05:06:19,465 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 05:06:21,280 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 05:06:23,282 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:06:24,446 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:06:28,283 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:06:28,623 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 05:06:28,624 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 05:06:33,777 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:06:34,456 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:06:38,777 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:06:39,457 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:06:43,634 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 05:06:43,639 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 05:06:44,726 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:06:48,906 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 05:06:48,908 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 05:06:48,909 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 05:06:48,910 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 05:06:49,462 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:06:49,488 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 05:06:49,489 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 05:06:49,911 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:06:51,283 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 05:06:54,463 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:06:55,286 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:06:58,629 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 05:06:58,630 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 05:07:00,722 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:07:04,535 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:07:04,535 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:07:05,723 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:07:09,536 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:07:10,723 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:07:13,639 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 05:07:13,643 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 05:07:15,782 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:07:18,981 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 05:07:18,983 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 05:07:18,984 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 05:07:18,985 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 05:07:19,515 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 05:07:19,516 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 05:07:19,541 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:07:20,987 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:07:21,287 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 05:07:24,542 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:07:26,290 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:07:28,635 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 05:07:28,636 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 05:07:31,757 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:07:34,543 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:07:36,758 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:07:39,544 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:07:41,758 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:07:43,646 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 05:07:43,652 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 05:07:47,757 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:07:49,033 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 05:07:49,035 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 05:07:49,035 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 05:07:49,036 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 05:07:49,547 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 05:07:49,547 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 05:07:49,552 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:07:51,291 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 05:07:53,293 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:07:54,552 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:07:58,294 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:07:58,639 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 05:07:58,639 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 05:08:03,723 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:08:03,783 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:08:08,724 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:08:08,784 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:08:08,784 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:08:13,655 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 05:08:13,661 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 05:08:13,813 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:08:18,425 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 05:08:18,427 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 05:08:18,427 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 05:08:18,429 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 05:08:18,582 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 05:08:18,830 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:08:18,831 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:08:19,430 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:08:19,583 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 05:08:21,295 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 05:08:23,831 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:08:23,832 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:08:25,298 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:08:28,645 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 05:08:28,646 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 05:08:30,787 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:08:33,833 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:08:33,833 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:08:35,788 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:08:38,834 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:08:38,834 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:08:40,788 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:08:43,653 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 05:08:43,654 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 05:08:45,820 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:08:48,056 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 05:08:48,058 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 05:08:48,059 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 05:08:48,060 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 05:08:48,606 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 05:08:48,837 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:08:49,607 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 05:08:51,062 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:08:51,300 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 05:08:53,839 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:08:56,303 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:08:58,646 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 05:08:58,647 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 05:09:01,811 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:09:03,840 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:09:06,811 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:09:08,841 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:09:11,812 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:09:13,652 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 05:09:13,654 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 05:09:17,740 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:09:17,905 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 05:09:17,907 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 05:09:17,907 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 05:09:17,908 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 05:09:18,633 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 05:09:18,850 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:09:19,635 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 05:09:21,303 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 05:09:23,305 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:09:23,850 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:09:23,851 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:09:28,306 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:09:28,648 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 05:09:28,649 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 05:09:33,793 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:09:33,852 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:09:33,853 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:09:38,793 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:09:38,853 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:09:38,853 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:09:43,648 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 05:09:43,651 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 05:09:44,736 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:09:47,896 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 05:09:47,898 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 05:09:47,898 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 05:09:47,900 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 05:09:48,661 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 05:09:48,860 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:09:49,662 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 05:09:49,901 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:09:51,306 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 05:09:53,861 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:09:55,308 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:09:58,648 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 05:09:58,649 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 05:10:00,745 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:10:03,863 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:10:05,746 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:10:08,864 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:10:10,746 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:10:13,648 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 05:10:13,651 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 05:10:16,725 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:10:17,267 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 05:10:17,269 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 05:10:17,269 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 05:10:17,271 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 05:10:17,686 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 05:10:18,687 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 05:10:18,869 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:10:21,310 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 05:10:22,312 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:10:23,869 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:10:27,313 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:10:28,649 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 05:10:28,649 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 05:10:32,750 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:10:33,987 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:10:37,751 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:10:38,987 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:10:42,756 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:10:43,652 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 05:10:43,653 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 05:10:46,695 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 05:10:46,697 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 05:10:46,698 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 05:10:46,699 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 05:10:46,716 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 05:10:48,700 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:10:48,718 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 05:10:49,082 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:10:49,083 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:10:51,314 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 05:10:54,083 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:10:54,084 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:10:54,316 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:10:58,649 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 05:10:58,649 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 05:10:59,714 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:11:04,315 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:11:04,316 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:11:04,715 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:11:09,316 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:11:09,317 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:11:09,715 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:11:13,652 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 05:11:13,652 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 05:11:14,802 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:11:17,362 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 05:11:17,362 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 05:11:17,363 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 05:11:17,364 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 05:11:17,740 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 05:11:18,742 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 05:11:19,324 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:11:20,365 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:11:21,318 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 05:11:24,325 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:11:26,320 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:11:28,652 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 05:11:28,653 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 05:11:31,765 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:11:34,327 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:11:36,766 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:11:39,328 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:11:39,328 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:11:41,766 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:11:43,657 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 05:11:43,664 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 05:11:47,558 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 05:11:47,560 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 05:11:47,560 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 05:11:47,560 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:11:47,561 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 05:11:47,782 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 05:11:49,336 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:11:50,784 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 05:11:51,322 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 05:11:53,324 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:11:54,337 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:11:58,324 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:11:58,654 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 05:11:58,654 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 05:12:03,735 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:12:04,339 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:12:08,735 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:12:09,340 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:12:13,654 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 05:12:13,654 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 05:12:13,748 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:12:18,757 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:12:19,351 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:12:20,789 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 05:12:20,790 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 05:12:20,791 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 05:12:20,792 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 05:12:20,808 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 05:12:21,326 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 05:12:22,810 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 05:12:24,328 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:12:24,349 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:12:28,654 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 05:12:28,655 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 05:12:29,815 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:12:34,350 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:12:34,816 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:12:39,351 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:12:39,816 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:12:43,654 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 05:12:43,655 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 05:12:45,766 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:12:49,360 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:12:50,772 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:12:51,331 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 05:12:53,423 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 05:12:53,425 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 05:12:53,426 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 05:12:53,428 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 05:12:53,834 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 05:12:54,358 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:12:54,835 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 05:12:56,429 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:12:58,654 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 05:12:58,655 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 05:13:01,738 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:13:04,359 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:13:06,739 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:13:09,360 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:13:11,740 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:13:13,654 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 05:13:13,655 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 05:13:16,782 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:13:19,362 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:13:21,345 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 05:13:22,349 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:13:24,371 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:13:25,600 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 05:13:25,602 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 05:13:25,602 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 05:13:25,604 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 05:13:25,869 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 05:13:26,870 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 05:13:27,604 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:13:28,654 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 05:13:28,655 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 05:13:32,762 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:13:34,371 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:13:37,763 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:13:39,371 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:13:42,763 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:13:43,654 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 05:13:43,655 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 05:13:47,777 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:13:49,373 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:13:51,341 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 05:13:53,342 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:13:53,513 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 05:13:53,515 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 05:13:53,515 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 05:13:53,520 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 05:13:53,888 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 05:13:54,380 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:13:54,381 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:13:54,888 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 05:13:58,522 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:13:58,654 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 05:13:58,655 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 05:14:03,721 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:14:04,382 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:14:04,382 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:14:08,721 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:14:09,383 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:14:09,383 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:14:13,655 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 05:14:13,655 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 05:14:13,772 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:14:18,778 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:14:19,393 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:14:20,384 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 05:14:20,386 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 05:14:20,387 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 05:14:20,388 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 05:14:20,919 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 05:14:21,342 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 05:14:22,920 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 05:14:24,344 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:14:24,530 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:14:28,655 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 05:14:28,655 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 05:14:29,756 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:14:34,532 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:14:34,757 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:14:39,533 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:14:39,757 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:14:43,655 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 05:14:43,655 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 05:14:44,788 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:14:49,546 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:14:49,792 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:14:51,344 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 05:14:52,141 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 05:14:52,142 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 05:14:52,143 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 05:14:52,144 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 05:14:52,944 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 05:14:54,555 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:14:54,556 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:14:54,946 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 05:14:55,146 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:14:58,655 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 05:14:58,656 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 05:15:00,765 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:15:04,557 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:15:04,557 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:15:05,766 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:15:09,558 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:15:09,558 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:15:10,766 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:15:13,655 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 05:15:13,656 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 05:15:15,768 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:15:19,570 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:15:19,571 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:15:20,781 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:15:21,350 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 05:15:22,944 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 05:15:22,945 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 05:15:22,945 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 05:15:22,947 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 05:15:22,976 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 05:15:24,567 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:15:24,977 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 05:15:25,948 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:15:28,655 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 05:15:28,656 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 05:15:31,785 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:15:34,568 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:15:36,786 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:15:39,569 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:15:41,786 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:15:43,655 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 05:15:43,656 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 05:15:46,795 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:15:49,578 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:15:51,350 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 05:15:52,354 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:15:54,577 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:15:54,613 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 05:15:54,615 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 05:15:54,616 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 05:15:54,617 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 05:15:55,005 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 05:15:57,006 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 05:15:57,618 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:15:58,656 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 05:15:58,657 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 05:16:02,767 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:16:04,578 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:16:07,768 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:16:09,579 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:16:12,768 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:16:13,656 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 05:16:13,657 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 05:16:17,828 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:16:19,581 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:16:21,357 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 05:16:23,361 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:16:24,586 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:16:25,633 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 05:16:25,634 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 05:16:25,635 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 05:16:25,637 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 05:16:26,028 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 05:16:27,029 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 05:16:28,638 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:16:28,656 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 05:16:28,656 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 05:16:33,768 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:16:34,588 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:16:38,769 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:16:39,588 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:16:43,656 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 05:16:43,657 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 05:16:43,811 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:16:48,812 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:16:49,590 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:16:51,357 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 05:16:54,369 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:16:54,603 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:16:56,279 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 05:16:56,280 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 05:16:56,281 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 05:16:56,282 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 05:16:57,062 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 05:16:58,656 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 05:16:58,657 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 05:16:59,063 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 05:16:59,791 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:17:03,660 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:17:04,792 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:17:08,798 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:17:08,799 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:17:09,793 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:17:13,656 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 05:17:13,657 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 05:17:15,730 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:17:18,800 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:17:20,735 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:17:21,367 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 05:17:23,810 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:17:25,169 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 05:17:25,170 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 05:17:25,171 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 05:17:25,173 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 05:17:26,088 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 05:17:26,174 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:17:27,088 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 05:17:28,656 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 05:17:28,657 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 05:17:31,820 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:17:33,807 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:17:36,820 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:17:38,808 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:17:41,821 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:17:43,656 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 05:17:43,657 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 05:17:47,751 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:17:48,809 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:17:51,374 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 05:17:53,378 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:17:53,819 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:17:55,054 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 05:17:55,056 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 05:17:55,056 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 05:17:55,057 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 05:17:55,114 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 05:17:57,115 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 05:17:58,657 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 05:17:58,657 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 05:17:58,748 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:18:03,748 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:18:03,819 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:18:03,819 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:18:08,749 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:18:08,819 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:18:13,657 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 05:18:13,657 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 05:18:13,784 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:18:18,785 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:18:18,821 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:18:21,376 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 05:18:23,829 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:18:24,378 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:18:25,535 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 05:18:25,537 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 05:18:25,537 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 05:18:25,539 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 05:18:26,141 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 05:18:27,142 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 05:18:28,657 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 05:18:28,657 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 05:18:29,766 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:18:33,936 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:18:33,936 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:18:34,766 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:18:38,937 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:18:38,937 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:18:39,767 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:18:43,657 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 05:18:43,658 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 05:18:44,806 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:18:49,015 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:18:49,015 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:18:49,807 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:18:51,374 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 05:18:54,019 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:18:55,383 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:18:58,080 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 05:18:58,082 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 05:18:58,082 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 05:18:58,087 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 05:18:58,166 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 05:18:58,657 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 05:18:58,658 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 05:18:59,167 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 05:19:00,733 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:19:04,023 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:19:05,733 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:19:09,024 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:19:10,734 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:19:13,657 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 05:19:13,658 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 05:19:16,728 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:19:19,424 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:19:21,379 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 05:19:22,381 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:19:24,434 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:19:27,387 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:19:28,663 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 05:19:28,664 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 05:19:29,956 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 05:19:29,958 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 05:19:29,958 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 05:19:29,963 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 05:19:30,197 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 05:19:31,199 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 05:19:32,964 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:19:34,433 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:19:37,964 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:19:39,434 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:19:42,965 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:19:43,661 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 05:19:43,662 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 05:19:48,744 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:19:49,466 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:19:49,466 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:19:51,383 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 05:19:54,386 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:19:54,467 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:19:54,467 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:19:58,664 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 05:19:58,664 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 05:19:59,809 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:20:02,277 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 05:20:02,278 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 05:20:02,278 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 05:20:02,281 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 05:20:03,226 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 05:20:03,680 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:20:03,680 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:20:05,227 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 05:20:05,282 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:20:08,681 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:20:08,681 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:20:10,283 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:20:13,664 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 05:20:13,664 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 05:20:15,778 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:20:18,901 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:20:20,778 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:20:21,386 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 05:20:23,902 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:20:26,392 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:20:28,674 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 05:20:28,675 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 05:20:31,274 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 05:20:31,276 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 05:20:31,277 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 05:20:31,278 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 05:20:32,253 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 05:20:32,279 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:20:33,254 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 05:20:33,909 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:20:37,280 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:20:38,910 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:20:42,282 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:20:43,800 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 05:20:43,801 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 05:20:47,919 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:20:48,911 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:20:51,391 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 05:20:53,393 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:20:53,912 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:20:58,402 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:20:58,712 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 05:20:58,713 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 05:21:00,195 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 05:21:00,196 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 05:21:00,197 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 05:21:00,198 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 05:21:00,284 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 05:21:01,285 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 05:21:03,918 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:21:04,200 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:21:08,919 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:21:09,200 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:21:13,709 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 05:21:13,710 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 05:21:14,844 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:21:18,920 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:21:19,845 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:21:21,394 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 05:21:23,921 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:21:25,398 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:21:28,716 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 05:21:28,717 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 05:21:29,983 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 05:21:29,984 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 05:21:29,985 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 05:21:29,986 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 05:21:30,313 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 05:21:30,987 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:21:31,314 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 05:21:33,928 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:21:35,988 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:21:38,928 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:21:40,988 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:21:43,714 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 05:21:43,715 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 05:21:46,847 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:21:48,930 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:21:51,397 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 05:21:52,399 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:21:53,930 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:21:57,404 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:21:58,721 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 05:21:58,722 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 05:22:01,516 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 05:22:01,517 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 05:22:01,517 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 05:22:01,519 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 05:22:02,346 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 05:22:02,519 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:22:03,347 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 05:22:03,940 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:22:07,520 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:22:08,941 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:22:12,520 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:22:13,717 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 05:22:13,717 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 05:22:17,824 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:22:18,960 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:22:21,401 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 05:22:23,403 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:22:23,960 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:22:28,409 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:22:28,727 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 05:22:28,728 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 05:22:31,524 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 05:22:31,526 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 05:22:31,526 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 05:22:31,528 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 05:22:32,369 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 05:22:33,370 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 05:22:33,530 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:22:33,973 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:22:38,531 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:22:38,974 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:22:43,531 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:22:43,723 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 05:22:43,723 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 05:22:48,812 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:22:48,976 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:22:51,405 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 05:22:53,976 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:22:54,408 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:22:58,729 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 05:22:58,733 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 05:22:59,894 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:23:01,675 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 05:23:01,677 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 05:23:01,677 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 05:23:01,678 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 05:23:02,395 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 05:23:03,396 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 05:23:04,079 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:23:05,679 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:23:09,080 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:23:10,679 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:23:13,725 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 05:23:13,726 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 05:23:15,807 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:23:19,082 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:23:20,807 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:23:21,410 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 05:23:24,082 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:23:26,413 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:23:28,730 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 05:23:28,731 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 05:23:31,897 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:23:32,295 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 05:23:32,297 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 05:23:32,297 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 05:23:32,299 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 05:23:32,432 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 05:23:33,433 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 05:23:34,090 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:23:37,300 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:23:39,091 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:23:42,301 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:23:43,727 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 05:23:43,728 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 05:23:47,819 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:23:49,093 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:23:51,413 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 05:23:53,416 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:23:54,094 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:23:58,424 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:23:58,728 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 05:23:58,728 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 05:24:03,108 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 05:24:03,109 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 05:24:03,110 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 05:24:03,111 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 05:24:03,459 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 05:24:04,100 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:24:04,113 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:24:05,461 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 05:24:09,100 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:24:09,114 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:24:13,728 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 05:24:13,729 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 05:24:14,870 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:24:19,102 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:24:19,870 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:24:21,417 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 05:24:24,103 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:24:25,420 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:24:28,734 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 05:24:28,738 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 05:24:30,854 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:24:32,903 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 05:24:32,904 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 05:24:32,904 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 05:24:32,906 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 05:24:33,492 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 05:24:34,308 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:24:34,309 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:24:35,493 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 05:24:35,907 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:24:39,309 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:24:40,908 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:24:43,731 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 05:24:43,732 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 05:24:46,828 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:24:49,311 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:24:49,312 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:24:51,421 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 05:24:52,424 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:24:54,312 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:24:54,312 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:24:57,424 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:24:58,731 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 05:24:58,732 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 05:25:02,888 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:25:04,320 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:25:05,423 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 05:25:05,425 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 05:25:05,425 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 05:25:05,427 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 05:25:05,522 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 05:25:07,523 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 05:25:08,428 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:25:09,318 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:25:13,429 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:25:13,731 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 05:25:13,732 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 05:25:18,849 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:25:19,320 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:25:21,426 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 05:25:24,321 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:25:24,429 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:25:28,731 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 05:25:28,732 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 05:25:29,835 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:25:34,336 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:25:34,423 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:25:34,839 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:25:35,151 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 05:25:35,153 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 05:25:35,153 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 05:25:35,154 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 05:25:35,549 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 05:25:37,550 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 05:25:39,548 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:25:39,548 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:25:40,155 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:25:43,732 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 05:25:43,732 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 05:25:45,902 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:25:49,549 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:25:49,549 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:25:50,902 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:25:51,429 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 05:25:54,550 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:25:54,551 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:25:56,432 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:25:58,732 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 05:25:58,732 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 05:26:01,917 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:26:04,566 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:26:05,006 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 05:26:05,007 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 05:26:05,007 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 05:26:05,008 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 05:26:05,579 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 05:26:05,579 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 05:26:07,009 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:26:09,561 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:26:12,010 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:26:13,732 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 05:26:13,732 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 05:26:17,796 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:26:19,563 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:26:21,434 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 05:26:23,436 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:26:24,563 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:26:28,444 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:26:28,737 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 05:26:28,738 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 05:26:33,856 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:26:33,888 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 05:26:33,890 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 05:26:33,890 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 05:26:33,891 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 05:26:34,570 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:26:34,606 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 05:26:35,607 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 05:26:38,893 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:26:39,571 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:26:43,735 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 05:26:43,736 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 05:26:44,879 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:26:49,724 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:26:49,724 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:26:49,879 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:26:51,438 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 05:26:54,725 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:26:54,725 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:26:55,440 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:26:58,743 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 05:26:58,747 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 05:27:00,838 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:27:04,174 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 05:27:04,175 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 05:27:04,175 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 05:27:04,176 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 05:27:04,637 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 05:27:04,735 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:27:04,736 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:27:05,638 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 05:27:06,177 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:27:09,736 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:27:09,736 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:27:11,178 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:27:13,740 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 05:27:13,741 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 05:27:16,832 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:27:18,752 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:27:18,752 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:27:21,442 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 05:27:22,444 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:27:23,752 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:27:23,753 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:27:27,452 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:27:28,740 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 05:27:28,740 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 05:27:32,575 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 05:27:32,577 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 05:27:32,577 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 05:27:32,577 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:27:32,578 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 05:27:32,661 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 05:27:33,662 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 05:27:33,981 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:27:33,981 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:27:37,579 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:27:38,982 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:27:38,982 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:27:42,580 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:27:43,740 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 05:27:43,741 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 05:27:47,890 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:27:48,984 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:27:48,984 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:27:51,446 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 05:27:53,449 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:27:53,985 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:27:53,985 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:27:58,454 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:27:58,751 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 05:27:58,757 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 05:28:02,957 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 05:28:02,959 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 05:28:02,959 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 05:28:02,961 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 05:28:03,687 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 05:28:03,688 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 05:28:03,962 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:28:04,359 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:28:04,360 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:28:08,962 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:28:09,360 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:28:09,361 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:28:13,744 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 05:28:13,745 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 05:28:14,845 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:28:19,363 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:28:19,363 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:28:19,846 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:28:21,450 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 05:28:24,364 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:28:24,364 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:28:25,452 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:28:28,744 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 05:28:28,745 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 05:28:30,830 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:28:34,372 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:28:34,765 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 05:28:34,767 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 05:28:34,767 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 05:28:34,769 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 05:28:35,717 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 05:28:35,718 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 05:28:36,770 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:28:39,372 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:28:41,770 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:28:43,744 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 05:28:43,745 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 05:28:46,839 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:28:49,374 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:28:51,454 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 05:28:52,456 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:28:54,375 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:28:57,457 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:28:58,744 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 05:28:58,745 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 05:29:02,898 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:29:03,527 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 05:29:03,528 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 05:29:03,529 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 05:29:03,530 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 05:29:03,740 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 05:29:04,380 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:29:05,742 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 05:29:08,531 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:29:09,381 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:29:13,532 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:29:13,745 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 05:29:13,745 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 05:29:18,819 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:29:19,382 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:29:21,458 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 05:29:24,383 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:29:24,460 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:29:28,752 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 05:29:28,754 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 05:29:29,915 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:29:32,065 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 05:29:32,066 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 05:29:32,067 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 05:29:32,069 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 05:29:32,773 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 05:29:33,774 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 05:29:34,476 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:29:35,070 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:29:39,477 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:29:39,477 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:29:40,071 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:29:43,751 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 05:29:43,752 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 05:29:45,911 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:29:49,563 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:29:50,912 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:29:51,462 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 05:29:54,564 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:29:54,564 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:29:56,468 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:29:58,751 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 05:29:58,751 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 05:30:00,987 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 05:30:00,989 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 05:30:00,990 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 05:30:00,991 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 05:30:01,804 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 05:30:01,804 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 05:30:01,992 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:30:04,571 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:30:06,993 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:30:09,572 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:30:11,994 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:30:13,751 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 05:30:13,752 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 05:30:17,851 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:30:19,588 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:30:19,588 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:30:21,466 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 05:30:23,468 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:30:24,593 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:30:28,474 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:30:28,756 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 05:30:28,756 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 05:30:29,919 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 05:30:29,920 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 05:30:29,920 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 05:30:29,921 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 05:30:30,832 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 05:30:31,833 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 05:30:33,923 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:30:34,594 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:30:38,923 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:30:39,594 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:30:43,756 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 05:30:43,757 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 05:30:44,868 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:30:48,838 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:30:49,868 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:30:51,470 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 05:30:53,842 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:30:55,477 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:30:57,834 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 05:30:57,835 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 05:30:57,836 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 05:30:57,838 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 05:30:57,860 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 05:30:58,756 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 05:30:58,757 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 05:30:59,862 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 05:31:00,873 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:31:03,982 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:31:05,873 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:31:08,982 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:31:10,874 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:31:13,756 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 05:31:13,757 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 05:31:15,900 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:31:18,984 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:31:20,901 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:31:21,485 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 05:31:23,991 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:31:26,495 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:31:27,067 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 05:31:27,068 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 05:31:27,068 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 05:31:27,070 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 05:31:27,885 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 05:31:27,886 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 05:31:28,756 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 05:31:28,757 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 05:31:31,824 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:31:33,991 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:31:36,825 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:31:39,042 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:31:39,043 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:31:41,825 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:31:43,756 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 05:31:43,757 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 05:31:47,820 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:31:49,044 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:31:49,044 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:31:51,477 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 05:31:53,482 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:31:54,051 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:31:56,539 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 05:31:56,541 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 05:31:56,541 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 05:31:56,547 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 05:31:56,911 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 05:31:57,912 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 05:31:58,548 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:31:58,757 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 05:31:58,757 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 05:32:03,831 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:32:04,052 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:32:08,832 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:32:09,053 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:32:13,757 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 05:32:13,757 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 05:32:13,962 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:32:18,963 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:32:19,054 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:32:21,491 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 05:32:24,066 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:32:24,496 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:32:26,743 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 05:32:26,745 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 05:32:26,745 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 05:32:26,750 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 05:32:26,953 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 05:32:27,954 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 05:32:28,757 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 05:32:28,758 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 05:32:29,915 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:32:34,143 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:32:34,144 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:32:34,916 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:32:39,144 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:32:39,144 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:32:39,916 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:32:43,757 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 05:32:43,758 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 05:32:45,903 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:32:49,210 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:32:49,210 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:32:50,904 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:32:51,488 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 05:32:54,218 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:32:56,497 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:32:58,763 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 05:32:58,763 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 05:32:59,749 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 05:32:59,750 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 05:32:59,751 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 05:32:59,756 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 05:32:59,984 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 05:33:00,985 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 05:33:01,756 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:33:04,222 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:33:04,223 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:33:06,757 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:33:09,223 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:33:09,223 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:33:11,757 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:33:13,760 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 05:33:13,761 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 05:33:16,872 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:33:19,295 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:33:19,295 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:33:21,493 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 05:33:22,495 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:33:24,296 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:33:27,504 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:33:28,768 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 05:33:28,769 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 05:33:32,168 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 05:33:32,170 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 05:33:32,170 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 05:33:32,173 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 05:33:33,014 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 05:33:33,174 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:33:34,562 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:33:34,562 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:33:35,016 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 05:33:38,175 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:33:39,562 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:33:39,563 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:33:43,175 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:33:43,765 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 05:33:43,766 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 05:33:48,911 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:33:49,564 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:33:51,496 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 05:33:54,498 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:33:54,565 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:33:58,772 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 05:33:58,775 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 05:33:59,925 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:34:02,012 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 05:34:02,014 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 05:34:02,014 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 05:34:02,015 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 05:34:02,043 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 05:34:04,575 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:34:04,575 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:34:05,016 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:34:05,045 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 05:34:09,576 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:34:09,576 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:34:10,017 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:34:13,769 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 05:34:13,770 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 05:34:15,871 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:34:19,577 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:34:19,578 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:34:20,871 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:34:21,500 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 05:34:24,578 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:34:24,578 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:34:26,502 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:34:28,775 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 05:34:28,775 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 05:34:31,904 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:34:32,970 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 05:34:32,971 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 05:34:32,972 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 05:34:32,973 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 05:34:33,071 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 05:34:34,585 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:34:35,073 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 05:34:36,974 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:34:39,585 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:34:41,975 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:34:43,772 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 05:34:43,773 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 05:34:47,929 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:34:49,587 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:34:51,504 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 05:34:53,506 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:34:54,587 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:34:58,513 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:34:58,777 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 05:34:58,778 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 05:35:01,104 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 05:35:01,106 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 05:35:01,106 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 05:35:01,108 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 05:35:02,093 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 05:35:03,094 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 05:35:04,109 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:35:04,599 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:35:04,600 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:35:09,110 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:35:09,601 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:35:09,601 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:35:13,996 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 05:35:13,997 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 05:35:14,138 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:35:19,138 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:35:19,602 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:35:19,603 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:35:21,508 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 05:35:24,511 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:35:24,603 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:35:24,603 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:35:28,909 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 05:35:28,910 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 05:35:30,046 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:35:32,660 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 05:35:32,661 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 05:35:32,661 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 05:35:32,662 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 05:35:33,120 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 05:35:34,612 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:35:35,122 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 05:35:35,663 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:35:39,613 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:35:40,663 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:35:43,906 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 05:35:43,907 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 05:35:46,075 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:35:49,614 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:35:51,076 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:35:51,513 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 05:35:54,615 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:35:56,515 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:35:58,909 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 05:35:58,910 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 05:36:02,032 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:36:02,650 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 05:36:02,651 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 05:36:02,651 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 05:36:02,653 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 05:36:03,151 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 05:36:04,623 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:36:04,624 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:36:05,152 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 05:36:07,654 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:36:09,624 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:36:09,624 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:36:12,655 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:36:13,906 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 05:36:13,907 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 05:36:17,985 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:36:19,626 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:36:19,626 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:36:21,516 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 05:36:23,519 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:36:24,627 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:36:28,525 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:36:28,912 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 05:36:28,912 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 05:36:32,995 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 05:36:32,997 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 05:36:32,998 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 05:36:32,999 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 05:36:33,187 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 05:36:34,000 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:36:34,636 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:36:35,188 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 05:36:39,001 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:36:39,637 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:36:43,908 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 05:36:43,909 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 05:36:44,062 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:36:49,062 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:36:49,639 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:36:51,520 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 05:36:54,523 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:36:54,639 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:36:58,912 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 05:36:58,912 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 05:37:00,051 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:37:01,832 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 05:37:01,834 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 05:37:01,834 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 05:37:01,836 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 05:37:02,217 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 05:37:03,218 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 05:37:04,647 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:37:05,837 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:37:09,647 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:37:10,837 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:37:13,912 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 05:37:13,913 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 05:37:16,055 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:37:19,649 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:37:21,056 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:37:21,524 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 05:37:24,650 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:37:26,527 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:37:28,917 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 05:37:28,921 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 05:37:32,064 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:37:32,462 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 05:37:32,463 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 05:37:32,463 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 05:37:32,464 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 05:37:33,240 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 05:37:34,860 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:37:34,860 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:37:35,242 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 05:37:37,465 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:37:39,861 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:37:39,861 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:37:42,466 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:37:43,915 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 05:37:43,916 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 05:37:47,988 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:37:49,862 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:37:49,863 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:37:51,528 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 05:37:53,530 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:37:54,863 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:37:58,531 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:37:58,915 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 05:37:58,916 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 05:38:04,007 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:38:04,751 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 05:38:04,752 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 05:38:04,752 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 05:38:04,753 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 05:38:05,155 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:38:05,156 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:38:05,269 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 05:38:07,271 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 05:38:09,754 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:38:10,156 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:38:13,915 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 05:38:13,916 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 05:38:15,052 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:38:19,158 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:38:19,158 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:38:20,052 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:38:21,532 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 05:38:24,159 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:38:24,159 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:38:25,535 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:38:28,915 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 05:38:28,916 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 05:38:31,052 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:38:34,167 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:38:36,062 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:38:37,920 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 05:38:37,922 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 05:38:37,922 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 05:38:37,924 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 05:38:38,299 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 05:38:39,300 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 05:38:39,316 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:38:39,317 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:38:41,926 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:38:43,915 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 05:38:43,916 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 05:38:47,083 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:38:49,318 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:38:51,537 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 05:38:52,539 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:38:54,318 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:38:57,540 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:38:58,916 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 05:38:58,916 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 05:39:03,084 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:39:04,331 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:39:08,088 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:39:09,325 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:39:09,955 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 05:39:09,956 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 05:39:09,956 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 05:39:09,958 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 05:39:10,328 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 05:39:11,329 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 05:39:13,916 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 05:39:13,917 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 05:39:14,057 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:39:19,058 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:39:19,364 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:39:19,364 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:39:21,540 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 05:39:24,365 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:39:24,365 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:39:24,542 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:39:28,916 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 05:39:28,916 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 05:39:30,057 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:39:34,367 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:39:34,367 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:39:35,057 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:39:39,375 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:39:40,068 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:39:42,172 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 05:39:42,174 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 05:39:42,174 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 05:39:42,176 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 05:39:42,364 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 05:39:43,365 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 05:39:43,916 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 05:39:43,917 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 05:39:45,082 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:39:49,907 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:39:49,907 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:39:50,082 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:39:51,544 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 05:39:54,908 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:39:54,908 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:39:55,547 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:39:58,916 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 05:39:58,917 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 05:40:01,080 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:40:04,909 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:40:04,910 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:40:06,080 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:40:09,917 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:40:11,091 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:40:12,990 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 05:40:12,991 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 05:40:12,991 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 05:40:12,993 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 05:40:13,384 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 05:40:13,916 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 05:40:13,917 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 05:40:15,386 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 05:40:17,084 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:40:18,918 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:40:21,548 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 05:40:22,550 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:40:23,918 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:40:27,551 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:40:28,916 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 05:40:28,917 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 05:40:32,997 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:40:33,923 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:40:33,923 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:40:38,000 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:40:38,935 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:40:42,545 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 05:40:42,546 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 05:40:42,547 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 05:40:42,548 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 05:40:43,409 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 05:40:43,549 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:40:43,916 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 05:40:43,917 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 05:40:45,410 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 05:40:48,933 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:40:49,015 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:40:51,552 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 05:40:53,933 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:40:54,555 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:40:58,916 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 05:40:58,917 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 05:41:00,035 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:41:03,935 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:41:05,035 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:41:07,425 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 05:41:10,939 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:41:14,932 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:41:15,836 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 05:41:15,836 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 05:41:15,958 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:41:20,958 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:41:21,599 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 05:41:23,586 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:41:23,669 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:41:26,601 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:41:31,602 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:41:32,179 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:41:32,314 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:41:32,962 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 05:41:32,963 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 05:41:37,080 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:41:40,908 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:41:41,283 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:41:42,081 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:41:47,081 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:41:50,316 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:41:51,235 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 05:41:51,236 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 05:41:51,558 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 05:41:52,560 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:41:57,560 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:42:00,028 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:42:00,290 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:42:02,561 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:42:05,504 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 05:42:05,505 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 05:42:05,650 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 05:42:07,652 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 05:42:07,671 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:42:10,547 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:42:10,547 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:42:12,671 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:42:15,548 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:42:15,548 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:42:17,672 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:42:20,504 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 05:42:20,505 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 05:42:21,562 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 05:42:23,564 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:42:25,550 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:42:28,565 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:42:30,550 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:42:33,566 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:42:35,504 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 05:42:35,505 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 05:42:38,667 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:42:40,552 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:42:43,667 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:42:45,553 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:42:48,668 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:42:50,504 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 05:42:50,505 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 05:42:51,570 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 05:42:54,573 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:42:55,554 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:42:59,573 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:43:04,574 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:43:09,575 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:43:14,576 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:43:18,966 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:43:18,971 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:43:18,971 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 05:43:18,972 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 05:43:20,106 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:43:21,570 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 05:43:23,972 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:43:23,972 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:43:25,572 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:43:28,973 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:43:28,973 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:43:30,573 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:43:33,967 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 05:43:33,967 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 05:43:36,119 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:43:38,975 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:43:38,975 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:43:41,120 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:43:43,983 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:43:43,983 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:43:46,120 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:43:48,967 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 05:43:48,967 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 05:43:51,574 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 05:43:51,580 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:43:53,985 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:43:53,985 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:43:56,576 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:43:58,986 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:44:01,577 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:44:03,967 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 05:44:03,967 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 05:44:07,061 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:44:08,987 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:44:12,062 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:44:13,988 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:44:17,062 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:44:18,967 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 05:44:18,968 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 05:44:21,579 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 05:44:22,581 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:44:23,989 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:44:27,582 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:44:28,990 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:44:32,583 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:44:33,967 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 05:44:33,968 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 05:44:38,070 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:44:38,991 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:44:43,071 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:44:43,992 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:44:43,992 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:44:48,072 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:44:48,967 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 05:44:48,968 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 05:44:51,582 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 05:44:53,585 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:44:53,994 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:44:53,994 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:44:58,585 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:44:58,995 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:44:58,996 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:45:03,586 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:45:03,968 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 05:45:03,968 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 05:45:08,997 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:45:09,139 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:45:14,139 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:45:19,140 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:45:21,586 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 05:45:24,589 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:45:29,589 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:45:34,590 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:45:39,590 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:45:44,591 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:45:47,922 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:45:47,926 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:45:47,926 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 05:45:47,927 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 05:45:49,979 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:45:51,590 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 05:45:51,622 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 05:45:52,930 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:45:52,930 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:45:55,533 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:45:55,625 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 05:45:57,931 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:45:57,931 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:46:00,534 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:46:02,923 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 05:46:02,923 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 05:46:06,030 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:46:07,933 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:46:07,933 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:46:11,031 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:46:12,933 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:46:12,934 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:46:16,031 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:46:17,928 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 05:46:17,935 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 05:46:21,084 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:46:21,598 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 05:46:22,941 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:46:23,281 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 05:46:23,283 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 05:46:23,283 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 05:46:23,285 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 05:46:23,649 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 05:46:25,650 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 05:46:26,286 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:46:27,941 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:46:31,286 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:46:32,925 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 05:46:32,926 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 05:46:37,022 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:46:37,943 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:46:37,943 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:46:42,022 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:46:42,943 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:46:47,023 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:46:47,925 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 05:46:47,926 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 05:46:51,601 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 05:46:52,602 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:46:52,950 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:46:55,393 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 05:46:55,394 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 05:46:55,395 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 05:46:55,400 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 05:46:55,676 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 05:46:57,677 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 05:46:57,951 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:46:57,951 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:46:58,400 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:47:02,925 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 05:47:02,926 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 05:47:04,075 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:47:07,953 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:47:07,953 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:47:09,075 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:47:12,954 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:47:12,955 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:47:14,076 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:47:17,926 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 05:47:17,926 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 05:47:20,010 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:47:21,610 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 05:47:22,961 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:47:23,729 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 05:47:23,730 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 05:47:23,731 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 05:47:23,733 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 05:47:24,708 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 05:47:25,709 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 05:47:25,734 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:47:28,136 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:47:28,136 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:47:30,734 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:47:32,926 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 05:47:32,926 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 05:47:36,023 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:47:38,138 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:47:38,138 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:47:41,023 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:47:43,139 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:47:43,139 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:47:46,024 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:47:47,933 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 05:47:47,934 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 05:47:51,026 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:47:51,611 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 05:47:52,824 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 05:47:52,826 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 05:47:52,827 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 05:47:52,828 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 05:47:53,227 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:47:53,737 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 05:47:53,738 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 05:47:56,829 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:47:58,229 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:47:58,229 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:48:01,830 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:48:02,930 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 05:48:02,931 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 05:48:07,030 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:48:08,230 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:48:08,231 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:48:12,031 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:48:13,231 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:48:13,232 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:48:17,032 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:48:17,930 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 05:48:17,932 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 05:48:21,613 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 05:48:22,614 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:48:23,067 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 05:48:23,069 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 05:48:23,069 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 05:48:23,074 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 05:48:23,474 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:48:23,475 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:48:23,774 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 05:48:25,776 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 05:48:28,076 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:48:28,475 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:48:32,931 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 05:48:32,931 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 05:48:34,073 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:48:38,477 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:48:39,074 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:48:43,478 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:48:43,478 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:48:44,074 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:48:47,931 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 05:48:47,933 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 05:48:50,037 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:48:51,620 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 05:48:52,577 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 05:48:52,579 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 05:48:52,579 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 05:48:52,581 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 05:48:52,803 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 05:48:53,487 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:48:53,804 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 05:48:55,582 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:48:58,488 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:49:00,583 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:49:02,931 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 05:49:02,931 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 05:49:06,072 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:49:08,579 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:49:08,579 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:49:11,073 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:49:13,580 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:49:13,580 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:49:16,073 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:49:17,936 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 05:49:17,936 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 05:49:21,621 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 05:49:21,622 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:49:22,582 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 05:49:22,584 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 05:49:22,584 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 05:49:22,585 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 05:49:22,834 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 05:49:23,730 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:49:23,730 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:49:23,835 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 05:49:27,587 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:49:28,731 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:49:28,731 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:49:32,587 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:49:32,932 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 05:49:32,933 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 05:49:38,018 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:49:38,733 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:49:38,733 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:49:43,019 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:49:43,734 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:49:43,734 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:49:47,943 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 05:49:47,944 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 05:49:48,077 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:49:51,623 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 05:49:52,389 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 05:49:52,390 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 05:49:52,391 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 05:49:52,392 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 05:49:52,857 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 05:49:53,393 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:49:53,811 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:49:53,811 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:49:53,858 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 05:49:58,394 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:49:58,812 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:49:58,812 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:50:02,940 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 05:50:02,941 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 05:50:04,032 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:50:08,814 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:50:08,814 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:50:09,032 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:50:13,814 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:50:13,815 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:50:14,033 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:50:17,940 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 05:50:17,941 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 05:50:20,030 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:50:21,628 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 05:50:23,819 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:50:25,633 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:50:26,276 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 05:50:26,278 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 05:50:26,278 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 05:50:26,282 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 05:50:26,890 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 05:50:27,891 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 05:50:28,821 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:50:31,284 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:50:32,940 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 05:50:32,941 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 05:50:37,026 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:50:38,823 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:50:42,027 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:50:43,823 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:50:47,028 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:50:47,940 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 05:50:47,941 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 05:50:51,633 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 05:50:52,637 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:50:53,835 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:50:57,642 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:50:57,757 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 05:50:57,759 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 05:50:57,759 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 05:50:57,764 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 05:50:57,912 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 05:50:58,903 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:50:58,904 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:50:59,914 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 05:51:02,766 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:51:02,940 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 05:51:02,941 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 05:51:08,072 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:51:08,905 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:51:13,073 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:51:13,906 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:51:17,941 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 05:51:17,941 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 05:51:18,109 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:51:21,640 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 05:51:23,641 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:51:23,913 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:51:25,847 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 05:51:25,848 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 05:51:25,849 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 05:51:25,851 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 05:51:25,941 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 05:51:27,943 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 05:51:28,852 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:51:28,993 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:51:32,941 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 05:51:32,941 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 05:51:34,112 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:51:37,994 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:51:37,994 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:51:39,113 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:51:42,995 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:51:42,995 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:51:44,113 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:51:47,941 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 05:51:47,941 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 05:51:50,031 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:51:51,653 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 05:51:52,996 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:51:55,655 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:51:56,887 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 05:51:56,889 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 05:51:56,890 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 05:51:56,891 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 05:51:56,967 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 05:51:57,968 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 05:51:58,038 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:51:58,038 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:52:00,893 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:52:02,941 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 05:52:02,942 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 05:52:06,011 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:52:08,039 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:52:08,040 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:52:11,012 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:52:13,041 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:52:13,041 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:52:16,013 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:52:17,941 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 05:52:17,942 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 05:52:21,024 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:52:21,647 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 05:52:23,048 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:52:26,653 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:52:26,795 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 05:52:26,796 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 05:52:26,797 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 05:52:26,801 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 05:52:26,997 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 05:52:27,998 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 05:52:28,206 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:52:31,802 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:52:32,941 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 05:52:32,942 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 05:52:37,100 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:52:38,207 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:52:38,208 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:52:42,101 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:52:43,213 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:52:43,213 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:52:47,102 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:52:47,941 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 05:52:47,942 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 05:52:51,648 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 05:52:52,650 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:52:53,215 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:52:53,215 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:52:57,662 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:52:58,226 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:52:59,400 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 05:52:59,402 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 05:52:59,402 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 05:52:59,408 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 05:53:00,026 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 05:53:00,027 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 05:53:02,942 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 05:53:02,942 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 05:53:03,087 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:53:08,087 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:53:08,221 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:53:13,088 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:53:13,235 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:53:13,235 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:53:17,942 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 05:53:17,942 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 05:53:19,026 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:53:21,652 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 05:53:23,236 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:53:24,654 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:53:28,244 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:53:29,659 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:53:31,454 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 05:53:31,455 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 05:53:31,455 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 05:53:31,458 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 05:53:32,056 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 05:53:32,057 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 05:53:32,942 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 05:53:32,943 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 05:53:35,023 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:53:38,243 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:53:40,024 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:53:43,243 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:53:45,024 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:53:47,942 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 05:53:47,943 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 05:53:51,013 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:53:51,656 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 05:53:53,245 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:53:56,666 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:53:58,249 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:54:01,385 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 05:54:01,387 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 05:54:01,387 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 05:54:01,389 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 05:54:02,087 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 05:54:02,087 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 05:54:02,390 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:54:02,942 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 05:54:02,943 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 05:54:08,106 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:54:08,252 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:54:13,106 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:54:13,253 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:54:17,942 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 05:54:17,943 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 05:54:19,030 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:54:21,660 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 05:54:23,255 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:54:24,662 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:54:28,263 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:54:29,674 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:54:32,484 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 05:54:32,485 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 05:54:32,486 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 05:54:32,491 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 05:54:32,942 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 05:54:32,943 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 05:54:33,115 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 05:54:34,117 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 05:54:35,085 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:54:38,263 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:54:40,085 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:54:43,264 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:54:45,086 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:54:47,942 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 05:54:47,943 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 05:54:51,010 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:54:51,664 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 05:54:53,266 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:54:56,674 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:54:58,274 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:54:58,275 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:55:01,678 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:55:02,465 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 05:55:02,467 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 05:55:02,467 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 05:55:02,469 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 05:55:02,943 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 05:55:02,943 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 05:55:03,141 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 05:55:04,142 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 05:55:07,062 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:55:08,277 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:55:12,062 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:55:13,278 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:55:17,063 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:55:17,943 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 05:55:17,943 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 05:55:21,669 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 05:55:22,671 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:55:23,351 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:55:27,671 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:55:28,352 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:55:32,677 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:55:32,944 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 05:55:32,944 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 05:55:37,881 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 05:55:37,883 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 05:55:37,883 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 05:55:37,884 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:55:37,889 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 05:55:38,174 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 05:55:38,360 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:55:38,361 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:55:40,176 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 05:55:42,890 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:55:43,361 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:55:43,361 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:55:47,891 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:55:47,944 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 05:55:47,944 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 05:55:51,672 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 05:55:53,499 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:55:53,674 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:55:58,500 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:55:58,674 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:56:02,952 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 05:56:02,952 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 05:56:04,123 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:56:08,360 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 05:56:08,362 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 05:56:08,362 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 05:56:08,365 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 05:56:08,759 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:56:08,760 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:56:09,203 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 05:56:09,367 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:56:10,204 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 05:56:13,760 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:56:14,367 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:56:17,946 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 05:56:17,946 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 05:56:20,033 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:56:21,675 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 05:56:23,762 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:56:23,762 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:56:25,677 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:56:28,763 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:56:28,763 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:56:30,678 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:56:32,946 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 05:56:32,947 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 05:56:36,031 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:56:38,773 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:56:40,487 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 05:56:40,489 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 05:56:40,490 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 05:56:40,493 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 05:56:41,237 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 05:56:41,494 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:56:42,238 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 05:56:44,547 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:56:44,547 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:56:46,495 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:56:47,946 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 05:56:47,947 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 05:56:51,679 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 05:56:51,681 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:56:53,548 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:56:53,548 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:56:56,681 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:56:58,549 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:56:58,549 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:57:01,682 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:57:02,946 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 05:57:02,947 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 05:57:07,120 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:57:08,567 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:57:11,480 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 05:57:11,481 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 05:57:11,481 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 05:57:11,487 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 05:57:12,269 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 05:57:12,487 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:57:13,270 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 05:57:14,393 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:57:14,393 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:57:17,488 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:57:17,946 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 05:57:17,947 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 05:57:21,683 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 05:57:22,685 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:57:23,395 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:57:23,395 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:57:27,686 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:57:28,396 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:57:28,396 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:57:32,686 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:57:32,947 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 05:57:32,947 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 05:57:38,108 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:57:38,405 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:57:42,872 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 05:57:42,873 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 05:57:42,874 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 05:57:42,879 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 05:57:43,298 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 05:57:43,405 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:57:43,880 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:57:45,300 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 05:57:47,947 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 05:57:47,948 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 05:57:49,061 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:57:51,686 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 05:57:53,406 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:57:54,689 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:57:58,407 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:57:59,689 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:58:02,947 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 05:58:02,947 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 05:58:05,103 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:58:08,417 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:58:10,110 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:58:13,415 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:58:13,980 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 05:58:13,981 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 05:58:13,981 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 05:58:13,986 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 05:58:14,337 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 05:58:15,338 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 05:58:15,987 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:58:17,947 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 05:58:17,947 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 05:58:21,086 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:58:21,690 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 05:58:23,419 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:58:26,692 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:58:28,420 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:58:31,693 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:58:32,947 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 05:58:32,948 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 05:58:37,059 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:58:38,425 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:58:42,066 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:58:43,428 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:58:43,550 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 05:58:43,552 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 05:58:43,553 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 05:58:43,554 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 05:58:44,361 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 05:58:45,363 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 05:58:47,555 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:58:47,947 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 05:58:47,948 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 05:58:51,694 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 05:58:52,696 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:58:53,429 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:58:57,697 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:58:58,431 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:59:02,698 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:59:02,947 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 05:59:02,948 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 05:59:08,092 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:59:08,442 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:59:12,283 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 05:59:12,284 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 05:59:12,285 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 05:59:12,286 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 05:59:12,391 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 05:59:13,287 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:59:13,393 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 05:59:13,875 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:59:13,875 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:59:17,947 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 05:59:17,948 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 05:59:19,109 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:59:21,698 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 05:59:23,876 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:59:23,876 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:59:24,700 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:59:28,877 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:59:28,878 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:59:29,701 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:59:32,948 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 05:59:32,948 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 05:59:35,046 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:59:38,889 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:59:40,051 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:59:43,184 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 05:59:43,185 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 05:59:43,186 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 05:59:43,187 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 05:59:43,420 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 05:59:43,888 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:59:45,187 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:59:45,422 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 05:59:47,948 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 05:59:47,949 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 05:59:51,061 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:59:51,702 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 05:59:53,890 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:59:56,704 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 05:59:59,077 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 05:59:59,078 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:00:01,705 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:00:02,948 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 06:00:02,948 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 06:00:07,016 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:00:08,079 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:00:08,079 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:00:12,027 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:00:13,085 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:00:16,044 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 06:00:16,046 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 06:00:16,046 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 06:00:16,047 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 06:00:16,449 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 06:00:17,048 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:00:17,450 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 06:00:17,948 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 06:00:17,948 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 06:00:21,706 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 06:00:22,709 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:00:23,190 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:00:23,190 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:00:27,709 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:00:28,191 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:00:28,191 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:00:32,710 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:00:32,949 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 06:00:32,949 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 06:00:38,095 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:00:38,192 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:00:38,192 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:00:43,106 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:00:43,198 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:00:47,086 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 06:00:47,088 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 06:00:47,088 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 06:00:47,090 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 06:00:47,483 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 06:00:47,948 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 06:00:47,949 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 06:00:49,102 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:00:49,484 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 06:00:51,710 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 06:00:53,232 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:00:54,712 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:00:58,233 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:00:59,713 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:01:02,948 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 06:01:02,949 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 06:01:05,045 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:01:08,234 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:01:10,046 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:01:13,239 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:01:15,053 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:01:16,563 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 06:01:16,565 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 06:01:16,565 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 06:01:16,567 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 06:01:17,513 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 06:01:17,513 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 06:01:17,948 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 06:01:17,949 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 06:01:20,056 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:01:21,711 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 06:01:23,243 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:01:25,713 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:01:28,244 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:01:30,714 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:01:32,949 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 06:01:32,950 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 06:01:36,083 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:01:38,245 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:01:41,091 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:01:43,252 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:01:46,089 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:01:46,517 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 06:01:46,519 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 06:01:46,519 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 06:01:46,521 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 06:01:46,535 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 06:01:47,536 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 06:01:47,949 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 06:01:47,949 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 06:01:51,102 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:01:51,715 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 06:01:53,254 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:01:56,718 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:01:58,254 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:02:01,718 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:02:02,949 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 06:02:02,949 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 06:02:07,042 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:02:08,295 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:02:12,043 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:02:13,299 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:02:16,919 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 06:02:16,921 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 06:02:16,922 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 06:02:16,924 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 06:02:17,569 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 06:02:17,925 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:02:17,949 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 06:02:17,950 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 06:02:19,570 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 06:02:21,718 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 06:02:23,322 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:02:23,719 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:02:28,323 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:02:28,720 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:02:32,949 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 06:02:32,950 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 06:02:34,114 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:02:38,324 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:02:39,115 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:02:43,327 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:02:44,124 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:02:46,297 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 06:02:46,299 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 06:02:46,299 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 06:02:46,300 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 06:02:46,594 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 06:02:47,595 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 06:02:47,949 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 06:02:47,950 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 06:02:50,077 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:02:51,722 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 06:02:53,443 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:02:55,724 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:02:58,444 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:03:00,725 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:03:02,949 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 06:03:02,950 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 06:03:06,066 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:03:08,445 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:03:11,067 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:03:13,452 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:03:16,071 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:03:17,921 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 06:03:17,922 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 06:03:17,923 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 06:03:17,924 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 06:03:18,322 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 06:03:18,323 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 06:03:18,624 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 06:03:19,625 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 06:03:21,468 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:03:21,727 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 06:03:23,452 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:03:26,729 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:03:28,452 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:03:31,730 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:03:33,068 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 06:03:33,069 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 06:03:37,158 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:03:38,509 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:03:42,161 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:03:43,515 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:03:47,096 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 06:03:47,097 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 06:03:47,097 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 06:03:47,099 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 06:03:47,652 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 06:03:48,068 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 06:03:48,069 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 06:03:48,170 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:03:49,654 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 06:03:51,730 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 06:03:53,516 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:03:53,732 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:03:58,517 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:03:58,732 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:04:03,069 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 06:04:03,069 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 06:04:04,237 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:04:08,530 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:04:09,237 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:04:13,540 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:04:14,243 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:04:17,049 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 06:04:17,050 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 06:04:17,051 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 06:04:17,052 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 06:04:17,684 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 06:04:18,071 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 06:04:18,071 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 06:04:19,685 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 06:04:20,175 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:04:21,734 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 06:04:23,540 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:04:25,736 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:04:28,541 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:04:30,737 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:04:33,070 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 06:04:33,071 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 06:04:36,179 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:04:38,574 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:04:41,180 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:04:43,579 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:04:46,184 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:04:47,664 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 06:04:47,665 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 06:04:47,666 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 06:04:47,667 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 06:04:47,710 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 06:04:48,073 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 06:04:48,074 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 06:04:49,712 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 06:04:51,226 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:04:51,737 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 06:04:53,587 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:04:56,740 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:04:58,588 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:05:01,741 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:05:03,073 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 06:05:03,073 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 06:05:07,147 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:05:08,590 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:05:12,149 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:05:13,605 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:05:16,343 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 06:05:16,345 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 06:05:16,345 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 06:05:16,347 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 06:05:16,735 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 06:05:17,348 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:05:17,737 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 06:05:18,073 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 06:05:18,073 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 06:05:21,742 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 06:05:22,744 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:05:23,750 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:05:27,745 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:05:28,828 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:05:32,746 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:05:33,073 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 06:05:33,073 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 06:05:38,181 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:05:38,839 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:05:43,184 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:05:43,847 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:05:44,450 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 06:05:44,452 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 06:05:44,452 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 06:05:44,454 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 06:05:44,770 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 06:05:45,771 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 06:05:48,073 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 06:05:48,074 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 06:05:48,230 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:05:51,747 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 06:05:53,749 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:05:53,855 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:05:58,750 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:05:58,856 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:06:03,073 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 06:06:03,074 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 06:06:04,178 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:06:08,858 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:06:09,178 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:06:13,860 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:06:14,185 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:06:17,643 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 06:06:17,645 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 06:06:17,645 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 06:06:17,647 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 06:06:17,796 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 06:06:18,073 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 06:06:18,074 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 06:06:19,231 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:06:19,797 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 06:06:21,749 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 06:06:24,047 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:06:24,047 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:06:24,752 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:06:29,048 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:06:29,752 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:06:33,073 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 06:06:33,074 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 06:06:35,158 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:06:39,049 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:06:40,158 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:06:44,050 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:06:45,168 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:06:48,076 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 06:06:48,077 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 06:06:48,821 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 06:06:48,823 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 06:06:48,823 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 06:06:48,825 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 06:06:49,822 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 06:06:49,822 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 06:06:50,826 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:06:51,753 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 06:06:53,226 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:06:53,227 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:06:56,756 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:06:58,228 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:06:58,387 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:07:01,756 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:07:03,075 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 06:07:03,076 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 06:07:07,210 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:07:08,388 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:07:08,389 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:07:12,210 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:07:13,389 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:07:13,390 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:07:17,214 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:07:18,080 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 06:07:18,083 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 06:07:20,078 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 06:07:20,079 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 06:07:20,080 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 06:07:20,085 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 06:07:20,848 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 06:07:21,757 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 06:07:21,850 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 06:07:22,760 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:07:23,485 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:07:23,486 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:07:27,760 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:07:28,486 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:07:28,487 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:07:32,761 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:07:33,077 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 06:07:33,077 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 06:07:38,162 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:07:38,520 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:07:38,520 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:07:43,162 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:07:43,521 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:07:43,521 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:07:48,084 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 06:07:48,089 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 06:07:48,195 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:07:51,765 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 06:07:52,329 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 06:07:52,330 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 06:07:52,331 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 06:07:52,332 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 06:07:52,883 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 06:07:53,333 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:07:53,739 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:07:53,740 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:07:53,884 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 06:07:58,334 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:07:58,740 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:08:03,080 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 06:08:03,081 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 06:08:04,163 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:08:08,742 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:08:09,163 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:08:13,743 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:08:13,743 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:08:14,164 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:08:18,081 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 06:08:18,081 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 06:08:19,207 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:08:21,763 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 06:08:22,275 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 06:08:22,276 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 06:08:22,277 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 06:08:22,280 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 06:08:22,909 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 06:08:23,752 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:08:23,910 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 06:08:24,281 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:08:28,753 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:08:29,282 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:08:33,081 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 06:08:33,082 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 06:08:35,229 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:08:38,755 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:08:40,229 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:08:43,755 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:08:45,230 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:08:48,081 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 06:08:48,081 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 06:08:51,206 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:08:51,767 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 06:08:52,076 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 06:08:52,077 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 06:08:52,078 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 06:08:52,079 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 06:08:52,939 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 06:08:53,763 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:08:53,940 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 06:08:57,081 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:08:58,764 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:09:02,082 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:09:03,081 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 06:09:03,082 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 06:09:07,201 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:09:08,765 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:09:08,766 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:09:12,201 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:09:13,766 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:09:13,766 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:09:17,202 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:09:18,081 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 06:09:18,082 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 06:09:21,774 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 06:09:22,778 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:09:23,775 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:09:26,225 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 06:09:26,226 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 06:09:26,227 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 06:09:26,232 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 06:09:26,965 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 06:09:27,966 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 06:09:28,233 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:09:28,774 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:09:28,774 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:09:33,081 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 06:09:33,082 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 06:09:34,160 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:09:38,857 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:09:39,161 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:09:43,857 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:09:43,858 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:09:44,162 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:09:48,082 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 06:09:48,082 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 06:09:50,164 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:09:51,778 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 06:09:53,863 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:09:55,283 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 06:09:55,284 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 06:09:55,285 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 06:09:55,285 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:09:55,286 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 06:09:55,994 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 06:09:55,994 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 06:09:58,864 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:10:00,288 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:10:03,082 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 06:10:03,082 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 06:10:06,191 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:10:08,938 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:10:11,191 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:10:13,938 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:10:16,192 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:10:18,082 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 06:10:18,082 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 06:10:21,786 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 06:10:21,790 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:10:23,946 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:10:25,859 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 06:10:25,861 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 06:10:25,862 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 06:10:25,863 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 06:10:26,016 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 06:10:26,864 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:10:28,018 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 06:10:29,265 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:10:29,265 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:10:31,865 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:10:33,082 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 06:10:33,082 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 06:10:37,177 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:10:38,266 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:10:38,266 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:10:42,178 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:10:43,267 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:10:43,267 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:10:47,178 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:10:48,082 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 06:10:48,083 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 06:10:51,787 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 06:10:52,793 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:10:53,275 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:10:56,341 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 06:10:56,343 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 06:10:56,343 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 06:10:56,344 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 06:10:57,042 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 06:10:58,043 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 06:10:58,279 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:10:58,279 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:10:58,345 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:11:03,082 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 06:11:03,083 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 06:11:04,152 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:11:08,280 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:11:08,281 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:11:09,153 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:11:13,282 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:11:13,282 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:11:14,153 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:11:18,082 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 06:11:18,083 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 06:11:19,249 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:11:21,792 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 06:11:23,288 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:11:24,726 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 06:11:24,728 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 06:11:24,729 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 06:11:24,729 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:11:24,730 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 06:11:25,064 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 06:11:26,065 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 06:11:28,289 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:11:28,289 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:11:29,732 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:11:33,082 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 06:11:33,083 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 06:11:35,237 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:11:38,382 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:11:40,237 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:11:43,382 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:11:43,382 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:11:45,238 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:11:48,082 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 06:11:48,083 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 06:11:51,204 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:11:51,799 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 06:11:53,390 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:11:56,370 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 06:11:56,371 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 06:11:56,371 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 06:11:56,371 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:11:56,372 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 06:11:57,090 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 06:11:58,092 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 06:11:58,517 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:12:01,373 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:12:03,082 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 06:12:03,083 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 06:12:07,227 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:12:08,518 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:12:08,519 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:12:12,227 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:12:13,520 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:12:13,520 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:12:17,228 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:12:18,083 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 06:12:18,083 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 06:12:21,798 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 06:12:22,799 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:12:23,523 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:12:25,946 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 06:12:25,948 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 06:12:25,949 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 06:12:25,954 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 06:12:26,118 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 06:12:27,956 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:12:28,120 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 06:12:28,524 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:12:32,956 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:12:33,083 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 06:12:33,083 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 06:12:38,223 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:12:38,526 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:12:43,224 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:12:43,527 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:12:48,083 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 06:12:48,083 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 06:12:48,229 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:12:51,803 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 06:12:53,539 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:12:53,806 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:12:55,932 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 06:12:55,933 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 06:12:55,934 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 06:12:55,935 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 06:12:56,150 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 06:12:58,152 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 06:12:58,535 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:12:58,936 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:13:03,083 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 06:13:03,084 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 06:13:04,241 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:13:08,537 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:13:09,242 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:13:13,537 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:13:14,242 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:13:18,083 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 06:13:18,084 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 06:13:20,178 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:13:21,808 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 06:13:23,543 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:13:25,489 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 06:13:25,490 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 06:13:25,491 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 06:13:25,491 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:13:25,493 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 06:13:26,180 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 06:13:26,181 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 06:13:28,642 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:13:28,642 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:13:30,494 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:13:33,083 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 06:13:33,084 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 06:13:36,180 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:13:38,643 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:13:41,180 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:13:43,644 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:13:46,181 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:13:48,083 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 06:13:48,084 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 06:13:51,204 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:13:51,812 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 06:13:53,653 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:13:56,822 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:13:58,057 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 06:13:58,059 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 06:13:58,060 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 06:13:58,061 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 06:13:58,207 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 06:13:58,652 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:14:00,209 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 06:14:02,063 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:14:03,083 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 06:14:03,084 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 06:14:07,240 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:14:08,654 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:14:12,241 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:14:13,654 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:14:17,241 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:14:18,084 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 06:14:18,084 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 06:14:21,817 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 06:14:22,824 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:14:23,660 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:14:27,828 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:14:27,951 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 06:14:27,953 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 06:14:27,954 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 06:14:27,955 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 06:14:28,237 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 06:14:28,661 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:14:30,238 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 06:14:32,956 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:14:33,084 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 06:14:33,084 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 06:14:38,195 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:14:38,662 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:14:43,195 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:14:43,755 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:14:48,084 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 06:14:48,085 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 06:14:48,230 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:14:51,821 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 06:14:53,756 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:14:53,756 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:14:53,832 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:14:58,717 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 06:14:58,719 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 06:14:58,720 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 06:14:58,722 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 06:14:59,128 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:14:59,128 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:14:59,260 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 06:14:59,722 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:15:00,261 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 06:15:03,084 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 06:15:03,085 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 06:15:05,253 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:15:08,129 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:15:10,253 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:15:13,305 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:15:13,305 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:15:15,254 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:15:18,084 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 06:15:18,085 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 06:15:21,239 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:15:21,825 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 06:15:23,306 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:15:26,839 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:15:28,315 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:15:28,938 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 06:15:28,940 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 06:15:28,940 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 06:15:28,942 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 06:15:29,287 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 06:15:30,288 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 06:15:31,943 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:15:33,084 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 06:15:33,085 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 06:15:37,186 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:15:38,341 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:15:38,341 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:15:42,186 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:15:43,435 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:15:43,436 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:15:47,187 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:15:48,084 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 06:15:48,085 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 06:15:51,829 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 06:15:52,831 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:15:53,437 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:15:53,437 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:15:57,846 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:15:58,441 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:15:59,875 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 06:15:59,876 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 06:15:59,877 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 06:15:59,878 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 06:16:00,320 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 06:16:02,322 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 06:16:02,879 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:16:03,084 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 06:16:03,085 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 06:16:08,187 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:16:08,442 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:16:13,188 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:16:13,567 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:16:13,568 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:16:18,084 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 06:16:18,085 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 06:16:18,244 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:16:21,832 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 06:16:23,569 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:16:23,570 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:16:23,837 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:16:28,580 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:16:28,841 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:16:29,558 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 06:16:29,559 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 06:16:29,560 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 06:16:29,561 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 06:16:30,349 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 06:16:30,350 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 06:16:33,085 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 06:16:33,085 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 06:16:34,196 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:16:38,704 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:16:38,704 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:16:39,197 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:16:43,705 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:16:43,705 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:16:44,198 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:16:48,085 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 06:16:48,085 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 06:16:49,207 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:16:51,838 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 06:16:53,706 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:16:53,707 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:16:54,840 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:16:58,712 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:16:59,845 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:17:01,244 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 06:17:01,245 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 06:17:01,245 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 06:17:01,247 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 06:17:01,374 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 06:17:02,375 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 06:17:03,085 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 06:17:03,086 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 06:17:05,156 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:17:08,715 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:17:10,157 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:17:13,716 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:17:15,157 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:17:18,085 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 06:17:18,086 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 06:17:20,169 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:17:21,841 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 06:17:23,717 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:17:25,852 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:17:28,726 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:17:30,718 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 06:17:30,718 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 06:17:30,719 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 06:17:30,721 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 06:17:31,401 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 06:17:31,722 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:17:32,402 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 06:17:33,085 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 06:17:33,086 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 06:17:37,212 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:17:38,864 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:17:42,212 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:17:43,865 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:17:43,865 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:17:47,213 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:17:48,085 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 06:17:48,086 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 06:17:51,845 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 06:17:52,847 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:17:53,866 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:17:53,866 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:17:57,850 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:17:58,876 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:18:01,850 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 06:18:01,852 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 06:18:01,852 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 06:18:01,854 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 06:18:02,437 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 06:18:02,437 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 06:18:02,855 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:18:03,085 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 06:18:03,086 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 06:18:08,239 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:18:09,000 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:18:13,239 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:18:14,054 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:18:14,142 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:18:18,086 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 06:18:18,087 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 06:18:19,215 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:18:21,849 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 06:18:23,143 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:18:24,851 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:18:28,151 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:18:29,857 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:18:31,592 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 06:18:31,593 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 06:18:31,594 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 06:18:31,596 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 06:18:32,464 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 06:18:32,464 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 06:18:33,086 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 06:18:33,086 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 06:18:35,252 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:18:38,152 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:18:40,253 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:18:43,153 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:18:45,253 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:18:48,086 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 06:18:48,086 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 06:18:51,193 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:18:51,853 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 06:18:53,154 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:18:56,862 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:18:58,161 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:19:01,862 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:19:02,750 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 06:19:02,751 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 06:19:02,752 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 06:19:02,753 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 06:19:03,157 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 06:19:03,158 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 06:19:03,486 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 06:19:04,487 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 06:19:07,297 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:19:08,163 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:19:12,298 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:19:13,163 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:19:13,164 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:19:17,299 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:19:18,155 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 06:19:18,156 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 06:19:21,857 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 06:19:22,860 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:19:23,165 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:19:23,165 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:19:27,860 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:19:28,166 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:19:32,866 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:19:33,167 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 06:19:33,171 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 06:19:35,577 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 06:19:35,579 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 06:19:35,579 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 06:19:35,581 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 06:19:36,522 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 06:19:36,522 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 06:19:38,171 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:19:38,582 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:19:43,172 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:19:43,583 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:19:48,160 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 06:19:48,161 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 06:19:49,310 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:19:51,861 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 06:19:53,300 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:19:53,300 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:19:54,863 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:19:58,303 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:19:59,872 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:20:03,064 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 06:20:03,066 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 06:20:03,067 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 06:20:03,068 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 06:20:03,533 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:20:03,537 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 06:20:03,537 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 06:20:03,553 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 06:20:05,555 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 06:20:05,624 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:20:09,534 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:20:09,535 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:20:10,625 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:20:14,535 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:20:14,536 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:20:15,626 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:20:18,210 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 06:20:18,211 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 06:20:21,378 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:20:21,865 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 06:20:23,537 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:20:23,537 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:20:26,867 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:20:28,538 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:20:28,538 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:20:31,871 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:20:33,210 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 06:20:33,211 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 06:20:35,602 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 06:20:35,604 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 06:20:35,605 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 06:20:35,606 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 06:20:36,584 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 06:20:37,585 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 06:20:37,607 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:20:38,543 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:20:42,608 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:20:43,544 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:20:47,609 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:20:48,210 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 06:20:48,211 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 06:20:51,869 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 06:20:52,871 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:20:53,545 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:20:57,871 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:20:58,546 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:21:02,875 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:21:03,220 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 06:21:03,221 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 06:21:05,524 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 06:21:05,525 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 06:21:05,525 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 06:21:05,526 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 06:21:05,614 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 06:21:07,616 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 06:21:08,527 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:21:08,678 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:21:08,678 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:21:13,528 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:21:13,678 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:21:13,679 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:21:18,214 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 06:21:18,215 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 06:21:19,295 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:21:21,873 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 06:21:23,680 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:21:23,680 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:21:24,875 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:21:28,688 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:21:29,881 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:21:33,214 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 06:21:33,214 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 06:21:33,663 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 06:21:33,665 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 06:21:33,665 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 06:21:33,667 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 06:21:34,639 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 06:21:35,640 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 06:21:35,668 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:21:38,813 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:21:40,668 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:21:43,814 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:21:43,814 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:21:45,669 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:21:48,214 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 06:21:48,215 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 06:21:51,302 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:21:51,877 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 06:21:53,816 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:21:53,816 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:21:56,884 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:21:58,832 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:21:58,833 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:22:01,272 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 06:22:01,274 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 06:22:01,274 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 06:22:01,276 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 06:22:01,666 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 06:22:02,277 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:22:03,214 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 06:22:03,215 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 06:22:03,668 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 06:22:07,297 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:22:08,829 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:22:12,297 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:22:13,947 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:22:14,038 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:22:17,298 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:22:18,215 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 06:22:18,215 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 06:22:21,882 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 06:22:22,884 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:22:24,040 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:22:24,040 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:22:27,893 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:22:29,053 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:22:30,973 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 06:22:30,974 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 06:22:30,975 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 06:22:30,976 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 06:22:31,693 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 06:22:32,977 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:22:33,215 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 06:22:33,215 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 06:22:33,694 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 06:22:38,385 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:22:39,126 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:22:43,385 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:22:44,126 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:22:44,127 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:22:48,215 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 06:22:48,216 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 06:22:49,319 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:22:51,885 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 06:22:54,128 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:22:54,128 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:22:54,887 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:22:59,139 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:22:59,895 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:23:02,231 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 06:23:02,232 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 06:23:02,233 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 06:23:02,234 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 06:23:02,718 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 06:23:03,215 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 06:23:03,216 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 06:23:03,719 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 06:23:05,382 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:23:09,136 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:23:09,137 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:23:10,383 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:23:14,137 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:23:14,137 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:23:15,383 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:23:18,215 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 06:23:18,216 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 06:23:21,337 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:23:21,888 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 06:23:24,138 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:23:26,890 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:23:29,143 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:23:31,904 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:23:33,215 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 06:23:33,216 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 06:23:33,564 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 06:23:33,565 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 06:23:33,566 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 06:23:33,567 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 06:23:33,748 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 06:23:35,750 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 06:23:37,569 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:23:39,148 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:23:42,569 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:23:44,149 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:23:47,570 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:23:48,215 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 06:23:48,216 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 06:23:51,892 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 06:23:52,894 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:23:54,150 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:23:54,151 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:23:57,896 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:23:59,163 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:24:02,902 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:24:03,215 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 06:24:03,216 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 06:24:03,273 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 06:24:03,378 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 06:24:03,379 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 06:24:03,381 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 06:24:03,775 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 06:24:05,777 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 06:24:08,383 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:24:09,161 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:24:13,383 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:24:14,161 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:24:18,216 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 06:24:18,216 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 06:24:19,379 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:24:21,896 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 06:24:24,162 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:24:24,898 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:24:29,167 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:24:29,904 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:24:33,215 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 06:24:33,216 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 06:24:34,149 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 06:24:34,151 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 06:24:34,151 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 06:24:34,153 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 06:24:34,807 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 06:24:35,154 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:24:35,808 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 06:24:38,296 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:24:40,155 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:24:43,297 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:24:43,297 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:24:45,155 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:24:48,216 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 06:24:48,216 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 06:24:50,321 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:24:51,901 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 06:24:53,299 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:24:53,299 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:24:55,903 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:24:58,303 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:25:00,912 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:25:03,216 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 06:25:03,216 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 06:25:03,644 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 06:25:03,646 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 06:25:03,646 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 06:25:03,648 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 06:25:03,838 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 06:25:05,840 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 06:25:06,649 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:25:08,305 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:25:11,650 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:25:13,305 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:25:16,650 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:25:18,216 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 06:25:18,217 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 06:25:21,904 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 06:25:21,911 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:25:23,307 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:25:26,907 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:25:28,319 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:25:31,914 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:25:33,219 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 06:25:33,221 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 06:25:33,558 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 06:25:33,560 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 06:25:33,561 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 06:25:33,562 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 06:25:33,868 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 06:25:35,870 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 06:25:37,564 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:25:38,316 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:25:42,564 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:25:43,316 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:25:47,565 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:25:48,219 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 06:25:48,220 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 06:25:51,908 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 06:25:52,911 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:25:53,318 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:25:57,911 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:25:58,319 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:26:02,919 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:26:03,229 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 06:26:03,229 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 06:26:06,626 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 06:26:06,628 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 06:26:06,629 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 06:26:06,630 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 06:26:06,897 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 06:26:07,898 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 06:26:08,324 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:26:08,631 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:26:13,324 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:26:13,632 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:26:18,221 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 06:26:18,222 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 06:26:19,319 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:26:21,912 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 06:26:23,326 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:26:24,915 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:26:28,326 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:26:29,916 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:26:33,226 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 06:26:33,227 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 06:26:35,340 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:26:38,334 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:26:38,628 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 06:26:38,630 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 06:26:38,631 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 06:26:38,632 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 06:26:38,925 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 06:26:39,926 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 06:26:40,633 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:26:43,335 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:26:45,634 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:26:48,224 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 06:26:48,225 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 06:26:51,381 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:26:51,917 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 06:26:53,458 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:26:53,458 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:26:56,920 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:26:58,459 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:26:58,459 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:27:01,921 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:27:03,224 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 06:27:03,225 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 06:27:07,394 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:27:08,468 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:27:09,762 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 06:27:09,764 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 06:27:09,765 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 06:27:09,767 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 06:27:09,952 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 06:27:11,954 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 06:27:12,768 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:27:13,468 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:27:13,468 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:27:17,768 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:27:18,224 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 06:27:18,225 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 06:27:21,920 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 06:27:22,922 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:27:23,470 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:27:23,470 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:27:27,923 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:27:28,470 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:27:32,924 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:27:33,235 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 06:27:33,236 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 06:27:38,338 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:27:38,479 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:27:38,877 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 06:27:38,879 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 06:27:38,879 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 06:27:38,881 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 06:27:38,986 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 06:27:41,989 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 06:27:43,480 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:27:43,882 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:27:48,227 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 06:27:48,228 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 06:27:49,397 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:27:51,925 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 06:27:53,481 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:27:53,481 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:27:54,927 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:27:58,482 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:27:58,483 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:27:59,928 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:28:03,228 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 06:28:03,228 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 06:28:05,354 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:28:08,495 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:28:08,495 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:28:10,355 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:28:10,520 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 06:28:10,522 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 06:28:10,522 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 06:28:10,524 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 06:28:11,013 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 06:28:12,014 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 06:28:13,489 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:28:13,489 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:28:15,525 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:28:18,227 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 06:28:18,228 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 06:28:21,385 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:28:21,928 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 06:28:23,490 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:28:26,931 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:28:28,538 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:28:28,538 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:28:31,931 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:28:33,227 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 06:28:33,228 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 06:28:37,303 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:28:38,545 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:28:42,312 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:28:43,383 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 06:28:43,384 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 06:28:43,385 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 06:28:43,386 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 06:28:43,785 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:28:43,786 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:28:44,040 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 06:28:44,041 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 06:28:47,387 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:28:48,227 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 06:28:48,228 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 06:28:51,932 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 06:28:52,934 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:28:53,787 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:28:57,935 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:28:58,788 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:29:02,935 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:29:03,228 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 06:29:03,228 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 06:29:08,315 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:29:08,791 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:29:08,791 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:29:13,321 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:29:13,798 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:29:15,376 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 06:29:15,378 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 06:29:15,379 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 06:29:15,380 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 06:29:16,070 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 06:29:16,071 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 06:29:18,228 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 06:29:18,229 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 06:29:18,342 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:29:21,936 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 06:29:23,799 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:29:23,800 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:29:23,938 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:29:28,909 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:29:28,910 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:29:28,939 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:29:33,228 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 06:29:33,229 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 06:29:34,370 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:29:38,920 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:29:39,380 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:29:43,916 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:29:44,383 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:29:44,470 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 06:29:44,472 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 06:29:44,473 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 06:29:44,474 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 06:29:45,113 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 06:29:46,115 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 06:29:48,228 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 06:29:48,229 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 06:29:50,318 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:29:51,940 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 06:29:53,918 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:29:53,918 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:29:55,942 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:29:59,090 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:29:59,091 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:30:00,943 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:30:03,228 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 06:30:03,229 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 06:30:06,386 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:30:09,096 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:30:09,096 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:30:11,388 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:30:14,102 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:30:15,661 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 06:30:15,663 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 06:30:15,663 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 06:30:15,665 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 06:30:16,148 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 06:30:16,666 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:30:18,150 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 06:30:18,229 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 06:30:18,229 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 06:30:21,947 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 06:30:21,953 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:30:24,102 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:30:26,951 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:30:29,109 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:30:29,110 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:30:31,952 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:30:33,228 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 06:30:33,229 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 06:30:37,342 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:30:39,114 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:30:42,350 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:30:44,117 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:30:44,117 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:30:44,282 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 06:30:44,284 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 06:30:44,284 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 06:30:44,286 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 06:30:45,176 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 06:30:46,177 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 06:30:48,229 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 06:30:48,229 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 06:30:48,338 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:30:51,948 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 06:30:53,950 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:30:54,118 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:30:54,118 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:30:58,950 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:30:59,182 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:30:59,182 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:31:03,229 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 06:31:03,229 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 06:31:04,391 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:31:09,186 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:31:09,401 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:31:13,890 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 06:31:13,892 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 06:31:13,892 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 06:31:13,894 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 06:31:14,206 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 06:31:14,310 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:31:14,311 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:31:14,895 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:31:16,208 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 06:31:18,229 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 06:31:18,229 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 06:31:20,412 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:31:21,952 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 06:31:23,312 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:31:23,312 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:31:25,954 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:31:28,313 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:31:28,313 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:31:30,955 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:31:33,229 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 06:31:33,229 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 06:31:36,388 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:31:38,315 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:31:38,315 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:31:41,395 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:31:43,319 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:31:45,526 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 06:31:45,527 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 06:31:45,528 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 06:31:45,529 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 06:31:46,241 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 06:31:46,241 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 06:31:46,530 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:31:48,229 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 06:31:48,230 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 06:31:51,956 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 06:31:51,962 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:31:53,327 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:31:53,327 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:31:56,959 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:31:58,328 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:31:58,328 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:32:01,959 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:32:03,229 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 06:32:03,230 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 06:32:07,402 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:32:08,340 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:32:12,405 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:32:12,877 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 06:32:12,878 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 06:32:12,878 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 06:32:12,879 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 06:32:13,270 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 06:32:13,338 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:32:14,271 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 06:32:17,880 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:32:18,229 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 06:32:18,230 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 06:32:21,960 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 06:32:22,962 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:32:23,339 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:32:27,962 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:32:28,340 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:32:32,963 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:32:33,229 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 06:32:33,230 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 06:32:38,341 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:32:38,375 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:32:43,348 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:32:43,388 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:32:44,377 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 06:32:44,379 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 06:32:44,379 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 06:32:44,381 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 06:32:45,295 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 06:32:46,297 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 06:32:48,229 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 06:32:48,230 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 06:32:49,343 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:32:51,964 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 06:32:53,347 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:32:54,967 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:32:58,472 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:32:58,555 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:32:59,967 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:33:03,230 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 06:33:03,230 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 06:33:05,395 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:33:08,556 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:33:10,402 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:33:13,563 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:33:15,404 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:33:15,591 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 06:33:15,593 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 06:33:15,593 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 06:33:15,595 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 06:33:16,321 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 06:33:16,322 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 06:33:18,230 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 06:33:18,230 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 06:33:21,397 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:33:21,967 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 06:33:23,562 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:33:26,970 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:33:28,563 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:33:31,971 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:33:33,230 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 06:33:33,230 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 06:33:37,306 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:33:38,564 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:33:42,313 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:33:43,573 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:33:46,832 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 06:33:46,833 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 06:33:46,834 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 06:33:46,835 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 06:33:47,353 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 06:33:47,836 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:33:48,230 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 06:33:48,230 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 06:33:48,354 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 06:33:51,970 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 06:33:52,972 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:33:53,570 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:33:57,973 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:33:58,571 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:34:02,973 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:34:03,230 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 06:34:03,231 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 06:34:08,316 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:34:08,572 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:34:13,316 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:34:13,573 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:34:18,234 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 06:34:18,235 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 06:34:18,394 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:34:20,163 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 06:34:20,164 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 06:34:20,164 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 06:34:20,166 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 06:34:20,387 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 06:34:21,974 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 06:34:22,389 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 06:34:23,586 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:34:23,976 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:34:28,587 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:34:28,977 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:34:33,231 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 06:34:33,232 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 06:34:34,372 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:34:38,588 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:34:39,373 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:34:43,589 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:34:44,378 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:34:48,233 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 06:34:48,239 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 06:34:49,598 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 06:34:49,599 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 06:34:49,599 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 06:34:49,600 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:34:49,601 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 06:34:50,414 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 06:34:50,415 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 06:34:51,978 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 06:34:53,594 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:34:54,980 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:34:58,594 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:34:59,981 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:35:03,233 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 06:35:03,234 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 06:35:05,370 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:35:08,706 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:35:10,370 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:35:13,707 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:35:15,379 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:35:18,238 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 06:35:18,246 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 06:35:21,035 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 06:35:21,036 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 06:35:21,037 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 06:35:21,037 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:35:21,038 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 06:35:21,441 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 06:35:21,982 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 06:35:22,442 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 06:35:23,718 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:35:23,719 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:35:26,985 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:35:28,719 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:35:31,985 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:35:33,236 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 06:35:33,236 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 06:35:37,387 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:35:38,720 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:35:38,721 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:35:42,393 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:35:43,722 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:35:47,398 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:35:47,641 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 06:35:47,641 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 06:35:47,642 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 06:35:47,644 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 06:35:48,236 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 06:35:48,237 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 06:35:48,466 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 06:35:48,467 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 06:35:51,987 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 06:35:52,989 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:35:54,050 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:35:57,990 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:35:59,092 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:36:02,991 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:36:03,236 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 06:36:03,236 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 06:36:08,333 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:36:09,098 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:36:13,339 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:36:14,102 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:36:14,868 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 06:36:14,870 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 06:36:14,870 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 06:36:14,872 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 06:36:15,488 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 06:36:16,489 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 06:36:18,236 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 06:36:18,237 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 06:36:18,373 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:36:21,990 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 06:36:23,279 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:36:23,279 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:36:23,992 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:36:28,369 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:36:28,370 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:36:28,993 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:36:33,237 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 06:36:33,237 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 06:36:34,385 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:36:38,371 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:36:39,385 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:36:43,379 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:36:44,396 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:36:46,261 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 06:36:46,263 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 06:36:46,263 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 06:36:46,265 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 06:36:46,517 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 06:36:48,243 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 06:36:48,243 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 06:36:48,519 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 06:36:50,391 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:36:51,994 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 06:36:53,409 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:36:53,409 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:36:55,996 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:36:58,409 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:36:58,410 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:37:00,997 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:37:03,243 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 06:37:03,244 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 06:37:06,503 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:37:08,411 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:37:08,411 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:37:11,503 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:37:13,423 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:37:16,511 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:37:18,246 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 06:37:18,246 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 06:37:19,114 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 06:37:19,116 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 06:37:19,116 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 06:37:19,118 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 06:37:19,543 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 06:37:20,544 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 06:37:21,998 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 06:37:22,004 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:37:23,517 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:37:23,517 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:37:27,001 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:37:28,518 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:37:28,518 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:37:32,001 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:37:33,246 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 06:37:33,246 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 06:37:37,337 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:37:38,520 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:37:38,520 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:37:42,338 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:37:43,521 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:37:43,521 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:37:47,348 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:37:48,250 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 06:37:48,250 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 06:37:50,250 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 06:37:50,251 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 06:37:50,252 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 06:37:50,254 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 06:37:50,568 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 06:37:52,002 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 06:37:52,569 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 06:37:53,004 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:37:53,667 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:37:53,667 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:37:58,006 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:37:58,668 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:37:58,668 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:38:03,006 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:38:03,250 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 06:38:03,251 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 06:38:08,390 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:38:08,670 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:38:08,670 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:38:13,390 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:38:13,671 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:38:13,671 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:38:18,254 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 06:38:18,255 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 06:38:18,410 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:38:22,007 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 06:38:23,262 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 06:38:23,264 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 06:38:23,265 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 06:38:23,267 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 06:38:23,602 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 06:38:23,680 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:38:24,267 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:38:24,603 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 06:38:28,681 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:38:29,268 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:38:33,251 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 06:38:33,252 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 06:38:34,406 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:38:38,682 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:38:39,407 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:38:43,683 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:38:44,408 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:38:48,261 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 06:38:48,262 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 06:38:49,430 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:38:52,009 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 06:38:52,200 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 06:38:52,201 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 06:38:52,202 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 06:38:52,207 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 06:38:52,631 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 06:38:52,631 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 06:38:53,688 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:38:55,208 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:38:58,689 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:39:00,209 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:39:03,255 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 06:39:03,256 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 06:39:05,395 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:39:08,691 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:39:10,396 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:39:13,692 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:39:15,397 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:39:18,256 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 06:39:18,256 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 06:39:20,422 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:39:22,020 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 06:39:23,702 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:39:24,274 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 06:39:24,276 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 06:39:24,276 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 06:39:24,282 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 06:39:24,659 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 06:39:24,659 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 06:39:26,283 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:39:28,700 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:39:31,283 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:39:33,255 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 06:39:33,256 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 06:39:36,408 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:39:38,701 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:39:41,409 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:39:43,702 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:39:46,410 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:39:48,255 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 06:39:48,256 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 06:39:51,415 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:39:52,018 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 06:39:53,720 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:39:55,035 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 06:39:55,036 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 06:39:55,036 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 06:39:55,039 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 06:39:55,687 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 06:39:56,688 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 06:39:57,040 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:39:58,715 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:40:02,040 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:40:03,256 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 06:40:03,256 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 06:40:07,392 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:40:08,717 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:40:12,393 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:40:13,717 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:40:17,393 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:40:18,261 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 06:40:18,261 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 06:40:22,025 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 06:40:23,027 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:40:23,477 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 06:40:23,479 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 06:40:23,480 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 06:40:23,481 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 06:40:23,714 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 06:40:23,881 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:40:24,715 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 06:40:28,482 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:40:28,881 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:40:33,261 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 06:40:33,262 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 06:40:34,410 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:40:38,883 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:40:39,411 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:40:43,883 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:40:44,411 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:40:48,267 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 06:40:48,279 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 06:40:50,402 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:40:52,025 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 06:40:53,890 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:40:53,891 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:40:53,980 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 06:40:53,981 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 06:40:53,982 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 06:40:53,984 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 06:40:54,744 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 06:40:54,745 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 06:40:55,985 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:40:58,891 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:41:00,985 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:41:03,262 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 06:41:03,263 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 06:41:06,428 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:41:08,892 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:41:11,428 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:41:13,893 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:41:16,429 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:41:18,262 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 06:41:18,263 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 06:41:22,037 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 06:41:22,041 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:41:23,899 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:41:24,297 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 06:41:24,299 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 06:41:24,299 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 06:41:24,300 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 06:41:24,772 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 06:41:24,773 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 06:41:27,301 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:41:28,900 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:41:32,301 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:41:33,262 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 06:41:33,263 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 06:41:37,374 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:41:38,901 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:41:42,375 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:41:43,902 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:41:47,376 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:41:48,262 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 06:41:48,263 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 06:41:52,040 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 06:41:53,041 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:41:53,907 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:41:56,088 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 06:41:56,090 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 06:41:56,091 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 06:41:56,092 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 06:41:56,802 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 06:41:56,803 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 06:41:58,094 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:41:58,907 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:42:03,094 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:42:03,262 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 06:42:03,263 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 06:42:08,348 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:42:08,908 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:42:13,348 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:42:13,909 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:42:18,263 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 06:42:18,263 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 06:42:18,354 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:42:22,038 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 06:42:23,917 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:42:24,041 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:42:25,974 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 06:42:25,975 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 06:42:25,976 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 06:42:25,979 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 06:42:26,833 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 06:42:26,833 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 06:42:28,916 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:42:29,980 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:42:33,263 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 06:42:33,263 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 06:42:35,365 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:42:38,917 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:42:40,365 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:42:44,087 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:42:45,366 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:42:48,263 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 06:42:48,264 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 06:42:50,389 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:42:52,039 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 06:42:54,089 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:42:56,047 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:42:59,093 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:43:00,489 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 06:43:00,491 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 06:43:00,491 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 06:43:00,495 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 06:43:00,855 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 06:43:01,496 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:43:01,856 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 06:43:03,263 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 06:43:03,263 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 06:43:07,424 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:43:09,098 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:43:12,425 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:43:14,134 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:43:14,219 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:43:17,425 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:43:18,263 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 06:43:18,264 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 06:43:22,043 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 06:43:23,045 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:43:24,220 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:43:28,051 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:43:29,230 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:43:31,469 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 06:43:31,470 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 06:43:31,471 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 06:43:31,475 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 06:43:31,883 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 06:43:33,263 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 06:43:33,264 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 06:43:33,353 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:43:33,885 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 06:43:38,354 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:43:39,223 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:43:43,354 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:43:44,223 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:43:48,263 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 06:43:48,264 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 06:43:48,408 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:43:52,047 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 06:43:54,050 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:43:54,224 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:43:59,057 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:43:59,232 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:44:01,831 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 06:44:01,833 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 06:44:01,833 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 06:44:01,835 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 06:44:01,910 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 06:44:03,263 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 06:44:03,264 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 06:44:03,912 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 06:44:04,423 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:44:09,235 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:44:09,423 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:44:14,236 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:44:14,424 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:44:18,264 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 06:44:18,264 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 06:44:20,367 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:44:22,051 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 06:44:24,237 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:44:26,054 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:44:29,249 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:44:31,058 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:44:32,337 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 06:44:32,339 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 06:44:32,340 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 06:44:32,345 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 06:44:32,943 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 06:44:33,269 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 06:44:33,269 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 06:44:33,944 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 06:44:36,369 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:44:39,247 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:44:39,248 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:44:41,370 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:44:44,248 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:44:44,249 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:44:46,371 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:44:48,264 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 06:44:48,265 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 06:44:52,055 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 06:44:52,057 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:44:54,250 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:44:54,250 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:44:57,063 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:44:59,267 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:45:02,062 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:45:02,418 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 06:45:02,420 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 06:45:02,420 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 06:45:02,426 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 06:45:02,975 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 06:45:03,264 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 06:45:03,265 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 06:45:03,976 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 06:45:07,358 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:45:09,263 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:45:09,263 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:45:12,359 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:45:14,264 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:45:14,264 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:45:17,359 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:45:18,265 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 06:45:18,265 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 06:45:22,058 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 06:45:23,060 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:45:23,266 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:45:23,266 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:45:28,060 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:45:28,267 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:45:28,267 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:45:33,067 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:45:33,273 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 06:45:33,276 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 06:45:34,754 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 06:45:34,756 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 06:45:34,756 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 06:45:34,762 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 06:45:35,016 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 06:45:36,018 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 06:45:38,279 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:45:38,279 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:45:38,763 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:45:43,280 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:45:43,280 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:45:43,764 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:45:48,268 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 06:45:48,269 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 06:45:49,716 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:45:52,059 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 06:45:53,282 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:45:53,282 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:45:55,061 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:45:58,292 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:46:00,066 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:46:03,270 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 06:46:03,270 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 06:46:03,699 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 06:46:03,703 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 06:46:03,703 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 06:46:03,704 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 06:46:04,049 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 06:46:05,705 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:46:06,051 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 06:46:08,289 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:46:10,706 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:46:13,290 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:46:15,706 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:46:18,270 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 06:46:18,271 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 06:46:21,434 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:46:22,064 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 06:46:23,291 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:46:27,066 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:46:28,292 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:46:32,070 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:46:33,276 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 06:46:33,277 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 06:46:34,936 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 06:46:34,938 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 06:46:34,939 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 06:46:34,940 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 06:46:35,080 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 06:46:36,081 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 06:46:37,942 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:46:38,347 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:46:38,347 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:46:42,942 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:46:43,348 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:46:43,348 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:46:47,943 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:46:48,370 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:46:48,371 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:46:48,374 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 06:46:48,374 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 06:46:52,068 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 06:46:53,071 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:46:54,372 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:46:58,072 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:46:59,372 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:46:59,373 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:47:03,082 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:47:03,283 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 06:47:03,284 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 06:47:04,363 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 06:47:04,364 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 06:47:04,365 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 06:47:04,366 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 06:47:05,103 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 06:47:06,104 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 06:47:08,368 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:47:08,513 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:47:13,368 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:47:13,514 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:47:18,280 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 06:47:18,281 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 06:47:18,407 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:47:22,071 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 06:47:23,515 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:47:24,074 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:47:28,516 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:47:29,074 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:47:33,290 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 06:47:33,294 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 06:47:34,435 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:47:35,350 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 06:47:35,351 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 06:47:35,351 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 06:47:35,354 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 06:47:36,136 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 06:47:38,138 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 06:47:38,550 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:47:40,404 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:47:43,550 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:47:45,405 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:47:48,283 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 06:47:48,284 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 06:47:51,395 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:47:52,075 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 06:47:53,775 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:47:57,078 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:47:58,776 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:48:02,090 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:48:03,289 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 06:48:03,293 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 06:48:05,633 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 06:48:05,635 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 06:48:05,636 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 06:48:05,639 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 06:48:06,168 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 06:48:07,640 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:48:08,170 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 06:48:09,039 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:48:09,039 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:48:12,641 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:48:14,039 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:48:14,040 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:48:17,641 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:48:18,287 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 06:48:18,288 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 06:48:22,079 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 06:48:23,082 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:48:24,117 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:48:24,117 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:48:28,082 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:48:29,121 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:48:33,090 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:48:33,288 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 06:48:33,289 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 06:48:34,553 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 06:48:34,555 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 06:48:34,555 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 06:48:34,557 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 06:48:35,192 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 06:48:36,194 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 06:48:38,558 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:48:39,124 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:48:43,559 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:48:44,124 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:48:48,288 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 06:48:48,288 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 06:48:49,450 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:48:52,083 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 06:48:53,299 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:48:55,086 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:48:58,306 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:49:00,089 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:49:03,291 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 06:49:03,291 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 06:49:03,957 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 06:49:03,959 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 06:49:03,960 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 06:49:03,961 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 06:49:04,224 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 06:49:05,962 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:49:06,226 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 06:49:08,357 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:49:08,357 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:49:10,963 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:49:13,358 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:49:13,358 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:49:15,963 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:49:18,288 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 06:49:18,289 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 06:49:21,373 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:49:22,088 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 06:49:23,490 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:49:23,490 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:49:27,091 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:49:28,491 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:49:28,491 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:49:32,101 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:49:33,291 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 06:49:33,299 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 06:49:35,354 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 06:49:35,356 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 06:49:35,356 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 06:49:35,357 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 06:49:36,251 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 06:49:37,358 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:49:38,252 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 06:49:38,781 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:49:38,781 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:49:42,359 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:49:43,782 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:49:43,783 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:49:47,359 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:49:48,289 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 06:49:48,290 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 06:49:52,091 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 06:49:53,094 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:49:53,788 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:49:53,788 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:49:58,094 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:49:58,788 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:49:58,789 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:50:03,103 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:50:03,294 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 06:50:03,294 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 06:50:06,196 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 06:50:06,198 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 06:50:06,198 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 06:50:06,200 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 06:50:06,276 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 06:50:08,201 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:50:08,278 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 06:50:08,793 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:50:13,201 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:50:13,794 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:50:18,202 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:50:18,395 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 06:50:18,395 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 06:50:22,097 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 06:50:23,795 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:50:24,099 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:50:28,796 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:50:29,101 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:50:33,315 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 06:50:33,319 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 06:50:34,459 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:50:36,336 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 06:50:36,337 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 06:50:36,338 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 06:50:36,340 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 06:50:37,306 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 06:50:38,307 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 06:50:38,802 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:50:40,341 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:50:43,803 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:50:45,341 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:50:48,310 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 06:50:48,311 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 06:50:50,477 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:50:52,099 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 06:50:53,804 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:50:56,102 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:50:58,805 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:51:01,111 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:51:03,311 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 06:51:03,311 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 06:51:06,461 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:51:06,493 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 06:51:06,495 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 06:51:06,495 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 06:51:06,497 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 06:51:07,339 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 06:51:08,340 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 06:51:08,900 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:51:08,901 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:51:11,498 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:51:13,901 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:51:16,499 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:51:18,311 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 06:51:18,312 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 06:51:22,104 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 06:51:22,110 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:51:23,903 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:51:27,106 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:51:28,903 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:51:32,107 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:51:33,312 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 06:51:33,312 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 06:51:37,403 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:51:38,910 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:51:39,333 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 06:51:39,335 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 06:51:39,335 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 06:51:39,336 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 06:51:39,367 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 06:51:40,368 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 06:51:43,337 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:51:43,921 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:51:43,921 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:51:48,311 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 06:51:48,312 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 06:51:48,382 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:51:52,107 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 06:51:53,922 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:51:54,110 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:51:58,923 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:51:59,110 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:52:03,312 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 06:52:03,312 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 06:52:04,442 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:52:08,934 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:52:09,451 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:52:10,969 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 06:52:10,971 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 06:52:10,971 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 06:52:10,972 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 06:52:11,394 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 06:52:12,395 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 06:52:14,180 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:52:14,973 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:52:18,312 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 06:52:18,312 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 06:52:20,463 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:52:22,111 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 06:52:24,181 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:52:26,113 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:52:29,222 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:52:31,114 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:52:33,312 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 06:52:33,312 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 06:52:36,471 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:52:39,229 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:52:41,481 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:52:42,583 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 06:52:42,584 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 06:52:42,585 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 06:52:42,586 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 06:52:43,421 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 06:52:44,228 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:52:44,228 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:52:44,422 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 06:52:46,587 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:52:48,312 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 06:52:48,312 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 06:52:52,114 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 06:52:52,120 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:52:54,229 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:52:54,230 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:52:57,117 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:52:59,230 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:52:59,231 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:53:02,117 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:53:03,312 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 06:53:03,312 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 06:53:07,436 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:53:09,245 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:53:10,760 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 06:53:10,761 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 06:53:10,761 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 06:53:10,762 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 06:53:11,448 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 06:53:12,449 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 06:53:12,763 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:53:14,762 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:53:14,763 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:53:17,764 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:53:18,312 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 06:53:18,313 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 06:53:22,118 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 06:53:23,120 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:53:23,763 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:53:23,764 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:53:28,121 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:53:28,764 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:53:28,765 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:53:33,122 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:53:33,312 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 06:53:33,313 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 06:53:38,463 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:53:38,769 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:53:42,394 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 06:53:42,395 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 06:53:42,395 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 06:53:42,397 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 06:53:42,479 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 06:53:43,870 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:53:44,398 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:53:44,480 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 06:53:48,312 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 06:53:48,313 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 06:53:49,477 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:53:52,122 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 06:53:53,871 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:53:53,872 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:53:55,125 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:53:58,873 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:53:58,873 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:54:00,125 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:54:03,312 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 06:54:03,313 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 06:54:05,411 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:54:08,879 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:54:10,420 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:54:12,167 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 06:54:12,169 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 06:54:12,169 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 06:54:12,171 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 06:54:12,507 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 06:54:13,882 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:54:14,509 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 06:54:16,172 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:54:18,313 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 06:54:18,313 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 06:54:21,414 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:54:22,126 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 06:54:23,884 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:54:27,129 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:54:28,885 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:54:32,129 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:54:33,313 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 06:54:33,313 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 06:54:37,481 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:54:38,894 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:54:41,963 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 06:54:41,964 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 06:54:41,965 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 06:54:41,966 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 06:54:42,533 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 06:54:42,968 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:54:43,970 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:54:44,535 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 06:54:47,968 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:54:48,313 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 06:54:48,313 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 06:54:52,131 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 06:54:53,133 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:54:53,971 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:54:58,134 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:54:58,972 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:54:58,972 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:55:03,134 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:55:03,313 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 06:55:03,313 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 06:55:08,440 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:55:08,984 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:55:13,446 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:55:13,978 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:55:14,571 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 06:55:14,573 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 06:55:14,574 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 06:55:14,575 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 06:55:15,562 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 06:55:16,563 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 06:55:18,313 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 06:55:18,314 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 06:55:19,417 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:55:22,134 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 06:55:23,980 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:55:25,136 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:55:28,981 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:55:30,137 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:55:33,313 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 06:55:33,314 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 06:55:35,475 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:55:38,983 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:55:40,482 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:55:43,993 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:55:45,261 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 06:55:45,262 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 06:55:45,263 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 06:55:45,264 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 06:55:45,600 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 06:55:46,265 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:55:46,601 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 06:55:48,313 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 06:55:48,314 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 06:55:51,443 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:55:52,137 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 06:55:53,991 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:55:57,139 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:55:58,991 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:56:02,140 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:56:03,314 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 06:56:03,314 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 06:56:07,480 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:56:08,997 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:56:12,292 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 06:56:12,294 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 06:56:12,295 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 06:56:12,296 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 06:56:12,631 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 06:56:13,297 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:56:14,001 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:56:14,633 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 06:56:18,298 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:56:18,313 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 06:56:18,314 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 06:56:22,141 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 06:56:24,003 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:56:24,143 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:56:29,004 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:56:29,144 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:56:33,314 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 06:56:33,314 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 06:56:34,382 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:56:39,006 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:56:39,383 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:56:44,017 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:56:44,388 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:56:45,509 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 06:56:45,510 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 06:56:45,511 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 06:56:45,513 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 06:56:45,662 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 06:56:46,663 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 06:56:48,314 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 06:56:48,314 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 06:56:49,410 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:56:52,145 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 06:56:54,016 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:56:54,016 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:56:55,147 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:56:59,017 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:56:59,017 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:57:00,148 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:57:03,314 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 06:57:03,314 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 06:57:05,437 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:57:09,018 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:57:09,019 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:57:10,441 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:57:14,028 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:57:15,445 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:57:16,267 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 06:57:16,268 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 06:57:16,269 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 06:57:16,271 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 06:57:16,696 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 06:57:18,314 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 06:57:18,315 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 06:57:18,697 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 06:57:20,478 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:57:22,149 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 06:57:24,026 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:57:26,151 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:57:29,027 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:57:31,152 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:57:33,314 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 06:57:33,315 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 06:57:36,463 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:57:39,028 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:57:41,464 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:57:44,036 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:57:46,472 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:57:48,315 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 06:57:48,315 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 06:57:48,738 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 06:57:48,740 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 06:57:48,740 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 06:57:48,742 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 06:57:49,723 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 06:57:50,724 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 06:57:51,743 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:57:52,153 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 06:57:54,148 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:57:57,156 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:57:59,149 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:58:02,156 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:58:03,315 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 06:58:03,316 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 06:58:07,445 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:58:09,150 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:58:09,151 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:58:12,446 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:58:14,155 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:58:17,457 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:58:18,315 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 06:58:18,315 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 06:58:18,778 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 06:58:18,780 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 06:58:18,780 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 06:58:18,781 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 06:58:19,748 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 06:58:20,749 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 06:58:22,157 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 06:58:23,159 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:58:24,187 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:58:28,160 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:58:29,188 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:58:29,188 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:58:33,161 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:58:33,315 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 06:58:33,316 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 06:58:38,491 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:58:39,189 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:58:39,190 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:58:43,497 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:58:44,198 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:58:46,741 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 06:58:46,743 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 06:58:46,743 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 06:58:46,745 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 06:58:46,770 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 06:58:48,315 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 06:58:48,316 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 06:58:48,772 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 06:58:49,431 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:58:52,161 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 06:58:54,195 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:58:55,163 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:58:59,196 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:59:00,164 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:59:03,316 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 06:59:03,316 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 06:59:05,422 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:59:09,197 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:59:10,423 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:59:14,205 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:59:15,429 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:59:17,948 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 06:59:17,952 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 06:59:17,952 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 06:59:17,954 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 06:59:18,348 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 06:59:18,349 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 06:59:18,800 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 06:59:18,800 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 06:59:20,512 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:59:22,165 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 06:59:24,348 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:59:24,348 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:59:26,167 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:59:29,349 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:59:29,349 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:59:31,168 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:59:33,348 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 06:59:33,348 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 06:59:36,475 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:59:38,350 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:59:38,351 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:59:41,476 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:59:43,351 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:59:43,352 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:59:46,487 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:59:48,354 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 06:59:48,355 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 06:59:50,306 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 06:59:50,308 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 06:59:50,308 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 06:59:50,309 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 06:59:50,833 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 06:59:52,169 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 06:59:52,175 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:59:52,835 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log +2024-12-26 06:59:53,452 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:59:53,453 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:59:57,171 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 06:59:58,453 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 06:59:58,454 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 07:00:02,172 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 07:00:03,351 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 07:00:03,352 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 07:00:07,458 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 07:00:08,455 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 07:00:08,456 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 07:00:12,458 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 07:00:13,456 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 07:00:13,456 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 07:00:17,461 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 07:00:18,359 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: stop_status +2024-12-26 07:00:18,360 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: stop_status +2024-12-26 07:00:22,136 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: partial_history +2024-12-26 07:00:22,138 DEBUG SenderThread:316822 [sender.py:send():379] send: history +2024-12-26 07:00:22,139 DEBUG SenderThread:316822 [sender.py:send_request():406] send_request: summary_record +2024-12-26 07:00:22,140 INFO SenderThread:316822 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-26 07:00:22,173 DEBUG SenderThread:316822 [sender.py:send():379] send: stats +2024-12-26 07:00:22,866 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/wandb-summary.json +2024-12-26 07:00:23,176 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: status_report +2024-12-26 07:00:23,539 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 07:00:23,539 DEBUG HandlerThread:316822 [handler.py:handle_request():158] handle_request: keepalive +2024-12-26 07:00:24,868 INFO Thread-12 :316822 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/balanced_prolong-512K-base_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241225_152744-sac86tlv/files/output.log